Who are you?

sz实例简析

发表于 | 分类于 | | 阅读次数

这篇文档主要对sz的example中的4个源代码进行简要分析学习，这里包括C接口的testdouble_compress.c和testdouble_decompress.c，以及Fortran接口的testdouble_compress.f90和testdouble_decompress.f90。

C 接口实例

压缩实例分析

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
/**
 *  @file test_compress.c
 *  @author Sheng Di
 *  @date April, 2015
 *  @brief This is an example of using compression interface
 *  (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */
#include <stdio.h>
#include <stdlib.h>
//sz所需的头文件 
#include "sz.h"
#include "rw.h"
//此部分内容用于计算time cost，可忽略
struct timeval startTime;
struct timeval endTime;  /* Start and end times */
struct timeval costStart; /*only used for recording the cost*/
double totalCost = 0;
void cost_start()
{
        gettimeofday(&costStart, NULL);
}
void cost_end()
{
        double elapsed;
        struct timeval costEnd;
        gettimeofday(&costEnd, NULL);
        elapsed = ((costEnd.tv_sec*1000000+costEnd.tv_usec)-(costStart.tv_sec*1000000+costStart.tv_usec))/1000000.0;
        totalCost += elapsed;
}
//此部分为主程序
int main(int argc, char * argv[])
{   
    //最高支持5维，不赋值默认为0
    int r5=0,r4=0,r3=0,r2=0,r1=0;
    //相关的文件路径，依次为：null，原始数据文件，压缩后数据文件
    char outDir[640], oriFilePath[640], outputFilePath[640];
    //sz配置文件
    char *cfgFile;
    
    //读取参数，main参数传递，参数个数少于3个报错
    if(argc < 3)
    {
	printf("Test case: testdouble_compress [config_file] [srcFilePath] [dimension sizes...]\n");
	printf("Example: testdouble_compress sz.config testdouble_8_8_128.dat 8 8 128\n");
	exit(0);
    }
   
    //指定配置文件
    cfgFile=argv[1];
    //格式化写入某个字符串，其原型：
    // int sprintf( char *buffer, const char *format, [ argument] … );
    sprintf(oriFilePath, "%s", argv[2]);
    //atoi() 函数用来将字符串转换成整数(int)，其原型为：
    // int atoi (const char * str);
    if(argc>=4)
    	r1 = atoi(argv[3]); //8
    if(argc>=5)
    	r2 = atoi(argv[4]); //8
    if(argc>=6)
	r3 = atoi(argv[5]); //128
    if(argc>=7)
    	r4 = atoi(argv[6]);
    if(argc>=8)
	r5 = atoi(argv[7]);
 
    printf("cfgFile=%s\n", cfgFile); 
  
    //压缩初始化
    int status = SZ_Init(cfgFile);
    //确定输出文件名
    sprintf(outputFilePath, "%s.sz", oriFilePath);
  
    //元素个数，out
    int nbEle;
    //将原多维数据文件读入内存中，同时将其一维化，实质是一维数组形式，具体的函数定义可看sz/src/rw.c文件
    double *data = readDoubleData(oriFilePath, &nbEle, &status);
    if(status!=SZ_SCES)
    {
	printf("Error: file %s cannot be read!\n", oriFilePath);
	exit(0);
    }
  
    //压缩后文件的大小，out
    int outSize;
    //计时开始
    cost_start(); 
    //压缩开始，
    unsigned char *bytes = SZ_compress(SZ_DOUBLE, data, &outSize, r5, r4, r3, r2, r1);
  
    //another compression method
    /*char *bytes = (char *)malloc(nbEle*sizeof(double)); //
    char* bytes = SZ_compress_args(SZ_DOUBLE, data, &outSize, ABS, 1E-12, 0.000001, r5, r4, r3, r2, r1);*/
  
    //计时结束
    cost_end();
    printf("timecost=%f\n",totalCost);
    //降压缩后的字节数据存入文件
    writeByteData(bytes, outSize, outputFilePath, &status);
    if(status!=SZ_SCES)
    {
	printf("Error: file %s cannot be written!\n", outputFilePath);
	free(data);
	exit(0);
    }
    //释放资源
    free(data);
    free(bytes);
    printf("done\n");
  
    //压缩结束
    SZ_Finalize();
    
    return 0;
}

解压缩实例分析

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
/**
 *  @file test_decompress.c
 *  @author Sheng Di
 *  @date April, 2015
 *  @brief This is an example of using Decompression interface.
 *  (C) 2015 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
//sz所需要的头文件
#include "sz.h"
#include "rw.h"
//此部分内容用于计算time cost，可忽略
struct timeval startTime;
struct timeval endTime;  /* Start and end times */
struct timeval costStart; /*only used for recording the cost*/
double totalCost = 0;
void cost_start()
{
        gettimeofday(&costStart, NULL);
}
void cost_end()
{
        double elapsed;
        struct timeval costEnd;
        gettimeofday(&costEnd, NULL);
        elapsed = ((costEnd.tv_sec*1000000+costEnd.tv_usec)-(costStart.tv_sec*1000000+costStart.tv_usec))/1000000.0;
        totalCost += elapsed;
}
//此部分为主程序
int main(int argc, char * argv[])
{
    //同上
    int r5=0,r4=0,r3=0,r2=0,r1=0;
    //表示：未压缩前的数据个数（r1×r2×...)，
    int nbEle, totalNbEle;
    //表示：压缩数据文件路径，解压缩后的文件路径
    char zipFilePath[640], outputFilePath[640];
    //同上
    char *cfgFile;
  
    //同上
    if(argc < 2)
    {
		printf("Test case: testdouble_decompress [configFile] [srcFilePath] [dimension sizes...]\n");
		printf("Example: testdouble_decompress sz.config testdouble_8_8_128.dat.sz 8 8 128\n");
		exit(0);
	}
    //同上
    cfgFile = argv[1];
    sprintf(zipFilePath, "%s", argv[2]);
    if(argc>=4)
        r1 = atoi(argv[3]); //8  
    if(argc>=5)
        r2 = atoi(argv[4]); //8
    if(argc>=6)
        r3 = atoi(argv[5]); //128  
    if(argc>=7)
        r4 = atoi(argv[6]);
    if(argc>=8)
        r5 = atoi(argv[7]);
    
    //确定未压缩前的元素个数
    if(r2==0)
        nbEle = r1;
    else if(r3==0)
        nbEle = r1*r2;
    else if(r4==0)
        nbEle = r1*r2*r3;
    else if(r5==0)
        nbEle = r1*r2*r3*r4;
    else
        nbEle = r1*r2*r3*r4*r5;
  
    //同上
    sprintf(outputFilePath, "%s.out", zipFilePath);
    //表示：压缩后的数据大小（字节计），程序执行状态，out
    int byteLength, status;
    unsigned char *bytes = readByteData(zipFilePath, &byteLength, &status);
    if(status!=SZ_SCES)
    {
        printf("Error: %s cannot be READ!\n", zipFilePath);
        exit(0);
    }
    
    //同上
    cost_start();
    //解压缩，全部为in
    double *data = SZ_decompress(SZ_DOUBLE, bytes, byteLength, r5, r4, r3, r2, r1);
    //同上
    cost_end();    
    printf("timecost=%f\n",totalCost);
    
    free(bytes); 
    //int i=0;
    //for(;i<8192;i++)
    //	printf("i=%d, data=%f\n",i,data[i]);
    
    //将解压后的数据（一维数据）写入文件
    writeDoubleData_inBytes(data, nbEle, outputFilePath, &status);
    if(status!=SZ_SCES)
    {
        printf("Error: %s cannot be written!\n", outputFilePath);
        exit(0);
    }
    
    printf("done\n");
    
    //SZ_Finalize();
    
    //以下用于与原数据对比，判断两者的差异程度
    char oriFilePath[640];
    strncpy(oriFilePath, zipFilePath, (unsigned)strlen(zipFilePath)-3);
    oriFilePath[strlen(zipFilePath)-3] = '\0';
    double *ori_data = readDoubleData(oriFilePath, &totalNbEle, &status);
    if(status!=SZ_SCES)
    {
        printf("Error: %s cannot be read!\n", oriFilePath);
        exit(0);
    }
    int i;
    double Max, Min, diffMax, err, maxpw_relerr = 0, relerr;
    Max = ori_data[0];
    Min = ori_data[0];
    diffMax = fabs(data[0] - ori_data[0]);
    for (i = 0; i < nbEle; i++)
    {
    	if (Max < ori_data[i]) Max = ori_data[i];
    	if (Min > ori_data[i]) Min = ori_data[i];
	err = fabs(data[i] - ori_data[i]);
    	if (diffMax < err)
    		diffMax = err;
        if(ori_data[i]!=0)
        {
                relerr = err/fabs(ori_data[i]);
		/*if(relerr>0.00001)
		{
			printf("error:i=%d, ori_data=%f, dec_data=%f\n",i, ori_data[i], data[i]);
			exit(0);
		}*/
                if(maxpw_relerr<relerr)
                        maxpw_relerr = relerr;
        }
	/*if(fabs(data[i] - ori_data[i]) > 1E-1)
	{
		printf("error: i=%d, %.20G, %.20G\n",i,ori_data[i], data[i]);
		exit(0);
	}*/
    }
    printf ("Max absolute error = %.20G\n", diffMax);
    printf ("Max relative error = %.20G\n", diffMax/(Max-Min));
    printf ("Max pw_relative err = %.20G\n", maxpw_relerr);
    free(ori_data);
    free(data);
    return 0;
}

Fortran 接口实例

压缩实例

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
program p
        ! 使用sz模块
        use sz
        use rw
        
        implicit none
        
        ! arg：配置文件参数
        character(len=32) :: arg
        
        ! allocatable：动态分配数组
        real(kind=8), dimension(:,:,:), allocatable :: grid
        integer(kind=4) :: gridsize1,gridsize2,gridsize3
        real(kind=8) :: res=0
        integer :: i,j,k
        integer(kind=4) :: ierr, outSize 
        
        ! the size of the compressed stream
        INTEGER(kind=1), DIMENSION(:), allocatable :: Bytes
        
        gridsize1 = 10 
        gridsize2 = 10 
        gridsize3 = 10 
        
        write (6,*) 'start....'
        allocate(grid(gridsize1,gridsize2,gridsize3))
        DO i=1,gridsize1
                DO j=1,gridsize2
                        DO k=1,gridsize3
                                grid(i,j,k)=i+j+k
                        END DO
                END DO
        END DO
     
        call getarg(1, arg)
        call SZ_Init(arg,ierr)
       
        call SZ_Compress(grid, Bytes, outSize)
        call writeData(Bytes, outSize, 'test_f.sz') 
         
        ! Free memory
        deallocate(grid)
        deallocate(Bytes)
        call SZ_Finalize()
        write (6,*) 'done.'
end program p

解压缩实例

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
program p
        !同上
        use sz
        use rw
        
        implicit none
        ! 配置文件
        character(len=32) :: arg
        ! 压缩后的数据流
        integer(kind=1), dimension(:), allocatable :: bytes
        ! 网格大小
        real(kind=8), dimension(:,:,:), allocatable :: grid
        integer(kind=4) :: gridsize1,gridsize2,gridsize3
        real(kind=8) :: res=0
        integer :: i,j,k
        integer(kind=4) :: ierr
        
        ! the size of the compressed stream
        integer(kind=4) :: outSize 
        
        gridsize1 = 10 
        gridsize2 = 10 
        gridsize3 = 10 
        
        write (6,*) 'start....'
     
        call getarg(1, arg)
        call SZ_Init(arg,ierr)
        
        ! 读入压缩数据，后两个参数都是out
        call readData('test_f.sz', bytes, outSize)
        
        ! 解压缩数据，（in，out，in，in，in）
        call SZ_Decompress(bytes, grid, gridsize1, gridsize2, gridsize3)
        
        ! 写入文件，从中可以看出out的grid数组是多维的
        open(unit=10,file='test_f.txt')
        DO i=1,gridsize3
                DO j=1,gridsize2
                        DO k=1,gridsize1
                                write (10,*) grid(k,j,i)
                        END DO
                END DO
        END DO
        
        deallocate(grid)
        deallocate(bytes) 
        write (6,*) 'done.'
        call SZ_Finalize()
end program p

参考：

  • [SZ_packages]/example/…