1 /* dfltcc.c -- compress data using IBM Z DEFLATE COMPRESSION CALL
2
3 Copyright (C) 2019-2023 Free Software Foundation, Inc.
4
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 3, or (at your option)
8 any later version.
9
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software Foundation,
17 Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */
18
19 #include <config.h>
20
21 #include <stdlib.h>
22
23 #ifdef HAVE_SYS_SDT_H
24 # include <sys/sdt.h>
25 #endif
26
27 #include "tailor.h"
28 #include "gzip.h"
29
30 #ifdef DYN_ALLOC
31 # error "DYN_ALLOC is not supported by DFLTCC"
32 #endif
33
34 /* ===========================================================================
35 * C wrappers for the DEFLATE CONVERSION CALL instruction.
36 */
37
38 typedef enum
39 {
40 DFLTCC_CC_OK = 0,
41 DFLTCC_CC_OP1_TOO_SHORT = 1,
42 DFLTCC_CC_OP2_TOO_SHORT = 2,
43 DFLTCC_CC_OP2_CORRUPT = 2,
44 DFLTCC_CC_AGAIN = 3,
45 } dfltcc_cc;
46
47 #define DFLTCC_QAF 0
48 #define DFLTCC_GDHT 1
49 #define DFLTCC_CMPR 2
50 #define DFLTCC_XPND 4
51 #define HBT_CIRCULAR (1 << 7)
52 #define DFLTCC_FACILITY 151
53 #define DFLTCC_FMT0 0
54 #define CVT_CRC32 0
55 #define HTT_FIXED 0
56 #define HTT_DYNAMIC 1
57
58 #ifndef DFLTCC_BLOCK_SIZE
59 # define DFLTCC_BLOCK_SIZE 1048576
60 #endif
61 #ifndef DFLTCC_FIRST_FHT_BLOCK_SIZE
62 # define DFLTCC_FIRST_FHT_BLOCK_SIZE 4096
63 #endif
64 #ifndef DFLTCC_LEVEL_MASK
65 # define DFLTCC_LEVEL_MASK 0x2
66 #endif
67 #ifndef DFLTCC_RIBM
68 # define DFLTCC_RIBM 0
69 #endif
70
71 #define MAX(a, b) ((a) > (b) ? (a) : (b))
72
73 #define STREQ(a, b) (strcmp (a, b) == 0)
74
75 struct dfltcc_qaf_param
76 {
77 char fns[16];
78 char reserved1[8];
79 char fmts[2];
80 char reserved2[6];
81 };
82
83 union aligned_dfltcc_qaf_param
84 {
85 struct dfltcc_qaf_param af;
86 char alignas (8) aligned;
87 };
88
89 struct dfltcc_param_v0
90 {
91 unsigned short pbvn; /* Parameter-Block-Version Number */
92 unsigned char mvn; /* Model-Version Number */
93 unsigned char ribm; /* Reserved for IBM use */
94 unsigned reserved32 : 31;
95 unsigned cf : 1; /* Continuation Flag */
96 unsigned char reserved64[8];
97 unsigned nt : 1; /* New Task */
98 unsigned reserved129 : 1;
99 unsigned cvt : 1; /* Check Value Type */
100 unsigned reserved131 : 1;
101 unsigned htt : 1; /* Huffman-Table Type */
102 unsigned bcf : 1; /* Block-Continuation Flag */
103 unsigned bcc : 1; /* Block Closing Control */
104 unsigned bhf : 1; /* Block Header Final */
105 unsigned reserved136 : 1;
106 unsigned reserved137 : 1;
107 unsigned dhtgc : 1; /* DHT Generation Control */
108 unsigned reserved139 : 5;
109 unsigned reserved144 : 5;
110 unsigned sbb : 3; /* Sub-Byte Boundary */
111 unsigned char oesc; /* Operation-Ending-Supplemental Code */
112 unsigned reserved160 : 12;
113 unsigned ifs : 4; /* Incomplete-Function Status */
114 unsigned short ifl; /* Incomplete-Function Length */
115 unsigned char reserved192[8];
116 unsigned char reserved256[8];
117 unsigned char reserved320[4];
118 unsigned short hl; /* History Length */
119 unsigned reserved368 : 1;
120 unsigned short ho : 15; /* History Offset */
121 unsigned int cv; /* Check Value */
122 unsigned eobs : 15; /* End-of-block Symbol */
123 unsigned reserved431 : 1;
124 unsigned char eobl : 4; /* End-of-block Length */
125 unsigned reserved436 : 12;
126 unsigned reserved448 : 4;
127 unsigned short cdhtl : 12; /* Compressed-Dynamic-Huffman Table
128 Length */
129 unsigned char reserved464[6];
130 unsigned char cdht[288];
131 unsigned char reserved[32];
132 unsigned char csb[1152];
133 };
134
135 union aligned_dfltcc_param_v0
136 {
137 struct dfltcc_param_v0 param;
138 char alignas (8) aligned;
139 };
140
141 static int
142 is_bit_set (const char *bits, int n)
143 {
144 return bits[n / 8] & (1 << (7 - (n % 8)));
145 }
146
147 static int
148 is_dfltcc_enabled (void)
149 {
150 char facilities[(DFLTCC_FACILITY / 64 + 1) * 8];
151
152 char const *env = getenv ("DFLTCC");
153 if (env && STREQ (env, "0"))
154 return 0;
155
156 memset (facilities, 0, sizeof facilities);
157 register char r0 __asm__ ("r0") = sizeof facilities / 8 - 1;
158 /* STFLE is supported since z9-109 and only in z/Architecture mode. When
159 * compiling with -m31, gcc defaults to ESA mode, however, since the kernel
160 * is 64-bit, it's always z/Architecture mode at runtime. */
161 __asm__ (
162 #ifndef __clang__
163 ".machinemode push\n"
164 ".machinemode zarch\n"
165 #endif
166 "stfle %[facilities]\n"
167 #ifndef __clang__
168 ".machinemode pop\n"
169 #endif
170 : [facilities] "=Q"(facilities), [r0] "+r"(r0) :: "cc");
171 return is_bit_set (facilities, DFLTCC_FACILITY);
172 }
173
174 static dfltcc_cc
175 dfltcc (int fn, void *param,
176 uch **op1, size_t *len1,
177 uch const **op2, size_t *len2,
178 void *hist)
179 {
180 uch *t2 = op1 ? *op1 : NULL;
181 size_t t3 = len1 ? *len1 : 0;
182 const uch *t4 = op2 ? *op2 : NULL;
183 size_t t5 = len2 ? *len2 : 0;
184 register int r0 __asm__ ("r0") = fn;
185 register void *r1 __asm__ ("r1") = param;
186 register uch *r2 __asm__ ("r2") = t2;
187 register size_t r3 __asm__ ("r3") = t3;
188 register const uch *r4 __asm__ ("r4") = t4;
189 register size_t r5 __asm__ ("r5") = t5;
190 int cc;
191
192 __asm__ volatile (
193 #ifdef HAVE_SYS_SDT_H
194 STAP_PROBE_ASM (zlib, dfltcc_entry,
195 STAP_PROBE_ASM_TEMPLATE (5))
196 #endif
197 ".insn rrf,0xb9390000,%[r2],%[r4],%[hist],0\n"
198 #ifdef HAVE_SYS_SDT_H
199 STAP_PROBE_ASM (zlib, dfltcc_exit,
200 STAP_PROBE_ASM_TEMPLATE (5))
201 #endif
202 "ipm %[cc]\n"
203 : [r2] "+r" (r2)
204 , [r3] "+r" (r3)
205 , [r4] "+r" (r4)
206 , [r5] "+r" (r5)
207 , [cc] "=r" (cc)
208 : [r0] "r" (r0)
209 , [r1] "r" (r1)
210 , [hist] "r" (hist)
211 #ifdef HAVE_SYS_SDT_H
212 , STAP_PROBE_ASM_OPERANDS (5, r2, r3, r4, r5, hist)
213 #endif
214 : "cc", "memory");
215 t2 = r2; t3 = r3; t4 = r4; t5 = r5;
216
217 if (op1)
218 *op1 = t2;
219 if (len1)
220 *len1 = t3;
221 if (op2)
222 *op2 = t4;
223 if (len2)
224 *len2 = t5;
225 return (cc >> 28) & 3;
226 }
227
228 static void
229 dfltcc_qaf (struct dfltcc_qaf_param *param)
230 {
231 dfltcc (DFLTCC_QAF, param, NULL, NULL, NULL, NULL, NULL);
232 }
233
234 static void
235 dfltcc_gdht (struct dfltcc_param_v0 *param)
236 {
237 const uch *next_in = inbuf + inptr;
238 size_t avail_in = insize - inptr;
239
240 dfltcc (DFLTCC_GDHT, param, NULL, NULL, &next_in, &avail_in, NULL);
241 }
242
243 static dfltcc_cc
244 dfltcc_cmpr_xpnd (struct dfltcc_param_v0 *param, int fn, off_t *total_in)
245 {
246 uch *next_out = outbuf + outcnt;
247 size_t avail_out = OUTBUFSIZ - outcnt;
248 const uch *next_in = inbuf + inptr;
249 size_t avail_in = insize - inptr;
250 dfltcc_cc cc = dfltcc (fn | HBT_CIRCULAR, param,
251 &next_out, &avail_out,
252 &next_in, &avail_in,
253 window);
254 off_t consumed_in = next_in - (inbuf + inptr);
255 inptr += consumed_in;
256 *total_in += consumed_in;
257 outcnt += ((OUTBUFSIZ - outcnt) - avail_out);
258 return cc;
259 }
260
261 static struct dfltcc_param_v0 *
262 init_param (union aligned_dfltcc_param_v0 *ctx)
263 {
264 char const *s = getenv ("DFLTCC_RIBM");
265 struct dfltcc_param_v0 *param = &ctx->param;
266 memset (param, 0, sizeof *param);
267 param->ribm = s && *s ? strtoul (s, NULL, 0) : DFLTCC_RIBM;
268 param->nt = 1;
269 param->cvt = CVT_CRC32;
270 param->cv = __builtin_bswap32 (getcrc ());
271 return param;
272 }
273
274 static void
275 bi_load (struct dfltcc_param_v0 *param)
276 {
277 bi_valid = param->sbb;
278 bi_buf = bi_valid == 0 ? 0 : outbuf[outcnt] & ((1 << bi_valid) - 1);
279 }
280
281 static void
282 bi_close_block (struct dfltcc_param_v0 *param)
283 {
284 bi_load (param);
285 send_bits (bi_reverse (param->eobs >> (15 - param->eobl), param->eobl),
286 param->eobl);
287 param->bcf = 0;
288 }
289
290 static void
291 close_block (struct dfltcc_param_v0 *param)
292 {
293 bi_close_block (param);
294 bi_windup ();
295 /* bi_windup has written out a possibly partial byte, fix up the position */
296 param->sbb = (param->sbb + param->eobl) % 8;
297 if (param->sbb != 0)
298 {
299 Assert (outcnt > 0, "outbuf must have enough space for EOBS");
300 outcnt--;
301 }
302 }
303
304 static void
305 close_stream (struct dfltcc_param_v0 *param)
306 {
307 if (param->bcf)
308 bi_close_block (param);
309 else
310 bi_load (param);
311 send_bits (1, 3); /* BFINAL=1, BTYPE=00 */
312 bi_windup ();
313 put_short (0x0000);
314 put_short (0xFFFF);
315 }
316
317 /* Compress ifd into ofd in hardware or fall back to software. */
318
319 int
320 dfltcc_deflate (int pack_level)
321 {
322 /* Check whether we can use hardware compression. */
323 if (!is_dfltcc_enabled () || getenv ("SOURCE_DATE_EPOCH"))
324 return deflate (pack_level);
325 char const *s = getenv ("DFLTCC_LEVEL_MASK");
326 unsigned long level_mask
327 = s && *s ? strtoul (s, NULL, 0) : DFLTCC_LEVEL_MASK;
328 if ((level_mask & (1 << pack_level)) == 0)
329 return deflate (pack_level);
330 union aligned_dfltcc_qaf_param ctx;
331 dfltcc_qaf (&ctx.af);
332 if (!is_bit_set (ctx.af.fns, DFLTCC_CMPR)
333 || !is_bit_set (ctx.af.fns, DFLTCC_GDHT)
334 || !is_bit_set (ctx.af.fmts, DFLTCC_FMT0))
335 return deflate (pack_level);
336
337 /* Initialize tuning parameters. */
338 s = getenv ("DFLTCC_BLOCK_SIZE");
339 unsigned long block_size
340 = s && *s ? strtoul (s, NULL, 0) : DFLTCC_BLOCK_SIZE;
341
342 s = getenv ("DFLTCC_FIRST_FHT_BLOCK_SIZE");
343 off_t block_threshold
344 = s && *s ? strtoul (s, NULL, 0) : DFLTCC_FIRST_FHT_BLOCK_SIZE;
345
346 union aligned_dfltcc_param_v0 ctx_v0;
347 struct dfltcc_param_v0 *param = init_param (&ctx_v0);
348 off_t total_in = 0;
349
350 /* Compress ifd into ofd in a loop. */
351 while (true)
352 {
353 /* Flush the output data. */
354 if (outcnt > OUTBUFSIZ - 8)
355 {
356 if (param->sbb == 0)
357 flush_outbuf ();
358 else
359 {
360 uch partial = outbuf[outcnt];
361 flush_outbuf ();
362 outbuf[outcnt] = partial;
363 }
364 }
365
366 /* Close the block. */
367 if (param->bcf && total_in == block_threshold && !param->cf)
368 {
369 close_block (param);
370 block_threshold += block_size;
371 }
372
373 /* Read the input data. */
374 if (inptr == insize)
375 {
376 if (fill_inbuf (1) == EOF && !param->cf)
377 break;
378 inptr = 0;
379 }
380
381 /* Temporarily mask some input data. */
382 int extra = MAX (0, total_in + (insize - inptr) - block_threshold);
383 insize -= extra;
384
385 /* Start a new block. */
386 if (!param->bcf)
387 {
388 if (total_in == 0 && block_threshold > 0)
389 param->htt = HTT_FIXED;
390 else
391 {
392 param->htt = HTT_DYNAMIC;
393 dfltcc_gdht (param);
394 }
395 }
396
397 /* Compress inbuf into outbuf. */
398 while (dfltcc_cmpr_xpnd (param, DFLTCC_CMPR, &total_in)
399 == DFLTCC_CC_AGAIN)
400 ;
401
402 /* Unmask the input data. */
403 insize += extra;
404
405 /* Continue the block */
406 param->bcf = 1;
407 }
408
409 close_stream (param);
410 setcrc (__builtin_bswap32 (param->cv));
411 return 0;
412 }
413
414 /* Decompress ifd into ofd in hardware or fall back to software. */
415 int
416 dfltcc_inflate ()
417 {
418 /* Check whether we can use hardware decompression. */
419 if (!is_dfltcc_enabled ())
420 return inflate ();
421 union aligned_dfltcc_qaf_param ctx;
422 dfltcc_qaf (&ctx.af);
423 if (!is_bit_set (ctx.af.fns, DFLTCC_XPND))
424 return inflate ();
425
426 union aligned_dfltcc_param_v0 ctx_v0;
427 struct dfltcc_param_v0 *param = init_param (&ctx_v0);
428 off_t total_in = 0;
429
430 /* Decompress ifd into ofd in a loop. */
431 while (true)
432 {
433 /* Perform I/O. */
434 if (outcnt == OUTBUFSIZ)
435 flush_outbuf ();
436 if (inptr == insize)
437 {
438 if (fill_inbuf (1) == EOF)
439 {
440 /* Premature EOF. */
441 return 2;
442 }
443 inptr = 0;
444 }
445
446 /* Decompress inbuf into outbuf. */
447 dfltcc_cc cc;
448 while ((cc = dfltcc_cmpr_xpnd (param, DFLTCC_XPND, &total_in))
449 == DFLTCC_CC_AGAIN)
450 ;
451 if (cc == DFLTCC_CC_OK)
452 {
453 /* The entire deflate stream has been successfully decompressed. */
454 break;
455 }
456 if (cc == DFLTCC_CC_OP2_CORRUPT && param->oesc != 0)
457 {
458 /* The deflate stream is corrupted. */
459 fprintf (stderr, "Operation-Ending-Supplemental Code 0x%x\n",
460 param->oesc);
461 flush_outbuf ();
462 return 2;
463 }
464 /* There must be more data to decompress. */
465 }
466
467 if (param->sbb != 0)
468 {
469 /* The deflate stream has ended in the middle of a byte. Go to
470 the next byte boundary, so that unzip can read CRC and length. */
471 inptr++;
472 }
473
474 /* Set CRC value and update bytes_out for unzip. */
475 setcrc (__builtin_bswap32 (param->cv));
476 flush_outbuf ();
477 return 0;
478 }