make_ext4fs: Remove off64_t in favor of standard off_t
[project/make_ext4fs.git] / libsparse / output_file.c
1 /*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define _FILE_OFFSET_BITS 64
18 #define _LARGEFILE64_SOURCE 1
19
20 #include <fcntl.h>
21 #include <inttypes.h>
22 #include <limits.h>
23 #include <stdbool.h>
24 #include <stddef.h>
25 #include <stdlib.h>
26 #include <string.h>
27 #include <sys/mman.h>
28 #include <sys/stat.h>
29 #include <sys/types.h>
30 #include <unistd.h>
31 #include <zlib.h>
32
33 #include "defs.h"
34 #include "output_file.h"
35 #include "sparse_crc32.h"
36 #include "sparse_format.h"
37
38 #define min(a, b) \
39 ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; })
40
41 #define SPARSE_HEADER_MAJOR_VER 1
42 #define SPARSE_HEADER_MINOR_VER 0
43 #define SPARSE_HEADER_LEN (sizeof(sparse_header_t))
44 #define CHUNK_HEADER_LEN (sizeof(chunk_header_t))
45
46 #define container_of(inner, outer_t, elem) \
47 ((outer_t *)((char *)inner - offsetof(outer_t, elem)))
48
49 struct output_file_ops {
50 int (*open)(struct output_file *, int fd);
51 int (*skip)(struct output_file *, int64_t);
52 int (*pad)(struct output_file *, int64_t);
53 int (*write)(struct output_file *, void *, int);
54 void (*close)(struct output_file *);
55 };
56
57 struct sparse_file_ops {
58 int (*write_data_chunk)(struct output_file *out, unsigned int len,
59 void *data);
60 int (*write_fill_chunk)(struct output_file *out, unsigned int len,
61 uint32_t fill_val);
62 int (*write_skip_chunk)(struct output_file *out, int64_t len);
63 int (*write_end_chunk)(struct output_file *out);
64 };
65
66 struct output_file {
67 int64_t cur_out_ptr;
68 unsigned int chunk_cnt;
69 uint32_t crc32;
70 struct output_file_ops *ops;
71 struct sparse_file_ops *sparse_ops;
72 int use_crc;
73 unsigned int block_size;
74 int64_t len;
75 char *zero_buf;
76 uint32_t *fill_buf;
77 char *buf;
78 };
79
80 struct output_file_gz {
81 struct output_file out;
82 gzFile gz_fd;
83 };
84
85 #define to_output_file_gz(_o) \
86 container_of((_o), struct output_file_gz, out)
87
88 struct output_file_normal {
89 struct output_file out;
90 int fd;
91 };
92
93 #define to_output_file_normal(_o) \
94 container_of((_o), struct output_file_normal, out)
95
96 struct output_file_callback {
97 struct output_file out;
98 void *priv;
99 int (*write)(void *priv, const void *buf, int len);
100 };
101
102 #define to_output_file_callback(_o) \
103 container_of((_o), struct output_file_callback, out)
104
105 static int file_open(struct output_file *out, int fd)
106 {
107 struct output_file_normal *outn = to_output_file_normal(out);
108
109 outn->fd = fd;
110 return 0;
111 }
112
113 static int file_skip(struct output_file *out, int64_t cnt)
114 {
115 off_t ret;
116 struct output_file_normal *outn = to_output_file_normal(out);
117
118 ret = lseek(outn->fd, cnt, SEEK_CUR);
119 if (ret < 0) {
120 error_errno("lseek");
121 return -1;
122 }
123 return 0;
124 }
125
126 static int file_pad(struct output_file *out, int64_t len)
127 {
128 int ret;
129 struct output_file_normal *outn = to_output_file_normal(out);
130
131 ret = ftruncate(outn->fd, len);
132 if (ret < 0) {
133 return -errno;
134 }
135
136 return 0;
137 }
138
139 static int file_write(struct output_file *out, void *data, int len)
140 {
141 int ret;
142 struct output_file_normal *outn = to_output_file_normal(out);
143
144 ret = write(outn->fd, data, len);
145 if (ret < 0) {
146 error_errno("write");
147 return -1;
148 } else if (ret < len) {
149 error("incomplete write");
150 return -1;
151 }
152
153 return 0;
154 }
155
156 static void file_close(struct output_file *out)
157 {
158 struct output_file_normal *outn = to_output_file_normal(out);
159
160 free(outn);
161 }
162
163 static struct output_file_ops file_ops = {
164 .open = file_open,
165 .skip = file_skip,
166 .pad = file_pad,
167 .write = file_write,
168 .close = file_close,
169 };
170
171 static int gz_file_open(struct output_file *out, int fd)
172 {
173 struct output_file_gz *outgz = to_output_file_gz(out);
174
175 outgz->gz_fd = gzdopen(fd, "wb9");
176 if (!outgz->gz_fd) {
177 error_errno("gzopen");
178 return -errno;
179 }
180
181 return 0;
182 }
183
184
185 static int gz_file_skip(struct output_file *out, int64_t cnt)
186 {
187 off_t ret;
188 struct output_file_gz *outgz = to_output_file_gz(out);
189
190 ret = gzseek(outgz->gz_fd, cnt, SEEK_CUR);
191 if (ret < 0) {
192 error_errno("gzseek");
193 return -1;
194 }
195 return 0;
196 }
197
198 static int gz_file_pad(struct output_file *out, int64_t len)
199 {
200 off_t ret;
201 struct output_file_gz *outgz = to_output_file_gz(out);
202
203 ret = gztell(outgz->gz_fd);
204 if (ret < 0) {
205 return -1;
206 }
207
208 if (ret >= len) {
209 return 0;
210 }
211
212 ret = gzseek(outgz->gz_fd, len - 1, SEEK_SET);
213 if (ret < 0) {
214 return -1;
215 }
216
217 gzwrite(outgz->gz_fd, "", 1);
218
219 return 0;
220 }
221
222 static int gz_file_write(struct output_file *out, void *data, int len)
223 {
224 int ret;
225 struct output_file_gz *outgz = to_output_file_gz(out);
226
227 ret = gzwrite(outgz->gz_fd, data, len);
228 if (ret < 0) {
229 error_errno("gzwrite");
230 return -1;
231 } else if (ret < len) {
232 error("incomplete gzwrite");
233 return -1;
234 }
235
236 return 0;
237 }
238
239 static void gz_file_close(struct output_file *out)
240 {
241 struct output_file_gz *outgz = to_output_file_gz(out);
242
243 gzclose(outgz->gz_fd);
244 free(outgz);
245 }
246
247 static struct output_file_ops gz_file_ops = {
248 .open = gz_file_open,
249 .skip = gz_file_skip,
250 .pad = gz_file_pad,
251 .write = gz_file_write,
252 .close = gz_file_close,
253 };
254
255 static int callback_file_open(struct output_file *out __unused, int fd __unused)
256 {
257 return 0;
258 }
259
260 static int callback_file_skip(struct output_file *out, int64_t off)
261 {
262 struct output_file_callback *outc = to_output_file_callback(out);
263 int to_write;
264 int ret;
265
266 while (off > 0) {
267 to_write = min(off, (int64_t)INT_MAX);
268 ret = outc->write(outc->priv, NULL, to_write);
269 if (ret < 0) {
270 return ret;
271 }
272 off -= to_write;
273 }
274
275 return 0;
276 }
277
278 static int callback_file_pad(struct output_file *out __unused, int64_t len __unused)
279 {
280 return -1;
281 }
282
283 static int callback_file_write(struct output_file *out, void *data, int len)
284 {
285 struct output_file_callback *outc = to_output_file_callback(out);
286
287 return outc->write(outc->priv, data, len);
288 }
289
290 static void callback_file_close(struct output_file *out)
291 {
292 struct output_file_callback *outc = to_output_file_callback(out);
293
294 free(outc);
295 }
296
297 static struct output_file_ops callback_file_ops = {
298 .open = callback_file_open,
299 .skip = callback_file_skip,
300 .pad = callback_file_pad,
301 .write = callback_file_write,
302 .close = callback_file_close,
303 };
304
305 int read_all(int fd, void *buf, size_t len)
306 {
307 size_t total = 0;
308 int ret;
309 char *ptr = buf;
310
311 while (total < len) {
312 ret = read(fd, ptr, len - total);
313
314 if (ret < 0)
315 return -errno;
316
317 if (ret == 0)
318 return -EINVAL;
319
320 ptr += ret;
321 total += ret;
322 }
323
324 return 0;
325 }
326
327 static int write_sparse_skip_chunk(struct output_file *out, int64_t skip_len)
328 {
329 chunk_header_t chunk_header;
330 int ret;
331
332 if (skip_len % out->block_size) {
333 error("don't care size %"PRIi64" is not a multiple of the block size %u",
334 skip_len, out->block_size);
335 return -1;
336 }
337
338 /* We are skipping data, so emit a don't care chunk. */
339 chunk_header.chunk_type = CHUNK_TYPE_DONT_CARE;
340 chunk_header.reserved1 = 0;
341 chunk_header.chunk_sz = skip_len / out->block_size;
342 chunk_header.total_sz = CHUNK_HEADER_LEN;
343 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
344 if (ret < 0)
345 return -1;
346
347 out->cur_out_ptr += skip_len;
348 out->chunk_cnt++;
349
350 return 0;
351 }
352
353 static int write_sparse_fill_chunk(struct output_file *out, unsigned int len,
354 uint32_t fill_val)
355 {
356 chunk_header_t chunk_header;
357 int rnd_up_len, count;
358 int ret;
359
360 /* Round up the fill length to a multiple of the block size */
361 rnd_up_len = ALIGN(len, out->block_size);
362
363 /* Finally we can safely emit a chunk of data */
364 chunk_header.chunk_type = CHUNK_TYPE_FILL;
365 chunk_header.reserved1 = 0;
366 chunk_header.chunk_sz = rnd_up_len / out->block_size;
367 chunk_header.total_sz = CHUNK_HEADER_LEN + sizeof(fill_val);
368 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
369
370 if (ret < 0)
371 return -1;
372 ret = out->ops->write(out, &fill_val, sizeof(fill_val));
373 if (ret < 0)
374 return -1;
375
376 if (out->use_crc) {
377 count = out->block_size / sizeof(uint32_t);
378 while (count--)
379 out->crc32 = sparse_crc32(out->crc32, &fill_val, sizeof(uint32_t));
380 }
381
382 out->cur_out_ptr += rnd_up_len;
383 out->chunk_cnt++;
384
385 return 0;
386 }
387
388 static int write_sparse_data_chunk(struct output_file *out, unsigned int len,
389 void *data)
390 {
391 chunk_header_t chunk_header;
392 int rnd_up_len, zero_len;
393 int ret;
394
395 /* Round up the data length to a multiple of the block size */
396 rnd_up_len = ALIGN(len, out->block_size);
397 zero_len = rnd_up_len - len;
398
399 /* Finally we can safely emit a chunk of data */
400 chunk_header.chunk_type = CHUNK_TYPE_RAW;
401 chunk_header.reserved1 = 0;
402 chunk_header.chunk_sz = rnd_up_len / out->block_size;
403 chunk_header.total_sz = CHUNK_HEADER_LEN + rnd_up_len;
404 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
405
406 if (ret < 0)
407 return -1;
408 ret = out->ops->write(out, data, len);
409 if (ret < 0)
410 return -1;
411 if (zero_len) {
412 ret = out->ops->write(out, out->zero_buf, zero_len);
413 if (ret < 0)
414 return -1;
415 }
416
417 if (out->use_crc) {
418 out->crc32 = sparse_crc32(out->crc32, data, len);
419 if (zero_len)
420 out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len);
421 }
422
423 out->cur_out_ptr += rnd_up_len;
424 out->chunk_cnt++;
425
426 return 0;
427 }
428
429 int write_sparse_end_chunk(struct output_file *out)
430 {
431 chunk_header_t chunk_header;
432 int ret;
433
434 if (out->use_crc) {
435 chunk_header.chunk_type = CHUNK_TYPE_CRC32;
436 chunk_header.reserved1 = 0;
437 chunk_header.chunk_sz = 0;
438 chunk_header.total_sz = CHUNK_HEADER_LEN + 4;
439
440 ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));
441 if (ret < 0) {
442 return ret;
443 }
444 out->ops->write(out, &out->crc32, 4);
445 if (ret < 0) {
446 return ret;
447 }
448
449 out->chunk_cnt++;
450 }
451
452 return 0;
453 }
454
455 static struct sparse_file_ops sparse_file_ops = {
456 .write_data_chunk = write_sparse_data_chunk,
457 .write_fill_chunk = write_sparse_fill_chunk,
458 .write_skip_chunk = write_sparse_skip_chunk,
459 .write_end_chunk = write_sparse_end_chunk,
460 };
461
462 static int write_normal_data_chunk(struct output_file *out, unsigned int len,
463 void *data)
464 {
465 int ret;
466 unsigned int rnd_up_len = ALIGN(len, out->block_size);
467
468 ret = out->ops->write(out, data, len);
469 if (ret < 0) {
470 return ret;
471 }
472
473 if (rnd_up_len > len) {
474 ret = out->ops->skip(out, rnd_up_len - len);
475 }
476
477 return ret;
478 }
479
480 static int write_normal_fill_chunk(struct output_file *out, unsigned int len,
481 uint32_t fill_val)
482 {
483 int ret;
484 unsigned int i;
485 unsigned int write_len;
486
487 /* Initialize fill_buf with the fill_val */
488 for (i = 0; i < out->block_size / sizeof(uint32_t); i++) {
489 out->fill_buf[i] = fill_val;
490 }
491
492 while (len) {
493 write_len = min(len, out->block_size);
494 ret = out->ops->write(out, out->fill_buf, write_len);
495 if (ret < 0) {
496 return ret;
497 }
498
499 len -= write_len;
500 }
501
502 return 0;
503 }
504
505 static int write_normal_skip_chunk(struct output_file *out, int64_t len)
506 {
507 return out->ops->skip(out, len);
508 }
509
510 int write_normal_end_chunk(struct output_file *out)
511 {
512 return out->ops->pad(out, out->len);
513 }
514
515 static struct sparse_file_ops normal_file_ops = {
516 .write_data_chunk = write_normal_data_chunk,
517 .write_fill_chunk = write_normal_fill_chunk,
518 .write_skip_chunk = write_normal_skip_chunk,
519 .write_end_chunk = write_normal_end_chunk,
520 };
521
522 void output_file_close(struct output_file *out)
523 {
524 out->sparse_ops->write_end_chunk(out);
525 out->ops->close(out);
526 }
527
528 static int output_file_init(struct output_file *out, int block_size,
529 int64_t len, bool sparse, int chunks, bool crc)
530 {
531 int ret;
532
533 out->len = len;
534 out->block_size = block_size;
535 out->cur_out_ptr = 0ll;
536 out->chunk_cnt = 0;
537 out->crc32 = 0;
538 out->use_crc = crc;
539
540 out->zero_buf = calloc(block_size, 1);
541 if (!out->zero_buf) {
542 error_errno("malloc zero_buf");
543 return -ENOMEM;
544 }
545
546 out->fill_buf = calloc(block_size, 1);
547 if (!out->fill_buf) {
548 error_errno("malloc fill_buf");
549 ret = -ENOMEM;
550 goto err_fill_buf;
551 }
552
553 if (sparse) {
554 out->sparse_ops = &sparse_file_ops;
555 } else {
556 out->sparse_ops = &normal_file_ops;
557 }
558
559 if (sparse) {
560 sparse_header_t sparse_header = {
561 .magic = SPARSE_HEADER_MAGIC,
562 .major_version = SPARSE_HEADER_MAJOR_VER,
563 .minor_version = SPARSE_HEADER_MINOR_VER,
564 .file_hdr_sz = SPARSE_HEADER_LEN,
565 .chunk_hdr_sz = CHUNK_HEADER_LEN,
566 .blk_sz = out->block_size,
567 .total_blks = out->len / out->block_size,
568 .total_chunks = chunks,
569 .image_checksum = 0
570 };
571
572 if (out->use_crc) {
573 sparse_header.total_chunks++;
574 }
575
576 ret = out->ops->write(out, &sparse_header, sizeof(sparse_header));
577 if (ret < 0) {
578 goto err_write;
579 }
580 }
581
582 return 0;
583
584 err_write:
585 free(out->fill_buf);
586 err_fill_buf:
587 free(out->zero_buf);
588 return ret;
589 }
590
591 static struct output_file *output_file_new_gz(void)
592 {
593 struct output_file_gz *outgz = calloc(1, sizeof(struct output_file_gz));
594 if (!outgz) {
595 error_errno("malloc struct outgz");
596 return NULL;
597 }
598
599 outgz->out.ops = &gz_file_ops;
600
601 return &outgz->out;
602 }
603
604 static struct output_file *output_file_new_normal(void)
605 {
606 struct output_file_normal *outn = calloc(1, sizeof(struct output_file_normal));
607 if (!outn) {
608 error_errno("malloc struct outn");
609 return NULL;
610 }
611
612 outn->out.ops = &file_ops;
613
614 return &outn->out;
615 }
616
617 struct output_file *output_file_open_callback(int (*write)(void *, const void *, int),
618 void *priv, unsigned int block_size, int64_t len,
619 int gz __unused, int sparse, int chunks, int crc)
620 {
621 int ret;
622 struct output_file_callback *outc;
623
624 outc = calloc(1, sizeof(struct output_file_callback));
625 if (!outc) {
626 error_errno("malloc struct outc");
627 return NULL;
628 }
629
630 outc->out.ops = &callback_file_ops;
631 outc->priv = priv;
632 outc->write = write;
633
634 ret = output_file_init(&outc->out, block_size, len, sparse, chunks, crc);
635 if (ret < 0) {
636 free(outc);
637 return NULL;
638 }
639
640 return &outc->out;
641 }
642
643 struct output_file *output_file_open_fd(int fd, unsigned int block_size, int64_t len,
644 int gz, int sparse, int chunks, int crc)
645 {
646 int ret;
647 struct output_file *out;
648
649 if (gz) {
650 out = output_file_new_gz();
651 } else {
652 out = output_file_new_normal();
653 }
654 if (!out) {
655 return NULL;
656 }
657
658 out->ops->open(out, fd);
659
660 ret = output_file_init(out, block_size, len, sparse, chunks, crc);
661 if (ret < 0) {
662 free(out);
663 return NULL;
664 }
665
666 return out;
667 }
668
669 /* Write a contiguous region of data blocks from a memory buffer */
670 int write_data_chunk(struct output_file *out, unsigned int len, void *data)
671 {
672 return out->sparse_ops->write_data_chunk(out, len, data);
673 }
674
675 /* Write a contiguous region of data blocks with a fill value */
676 int write_fill_chunk(struct output_file *out, unsigned int len,
677 uint32_t fill_val)
678 {
679 return out->sparse_ops->write_fill_chunk(out, len, fill_val);
680 }
681
682 int write_fd_chunk(struct output_file *out, unsigned int len,
683 int fd, int64_t offset)
684 {
685 int ret;
686 int64_t aligned_offset;
687 int aligned_diff;
688 int buffer_size;
689 char *ptr;
690
691 aligned_offset = offset & ~(4096 - 1);
692 aligned_diff = offset - aligned_offset;
693 buffer_size = len + aligned_diff;
694
695 char *data = mmap(NULL, buffer_size, PROT_READ, MAP_SHARED, fd,
696 aligned_offset);
697 if (data == MAP_FAILED) {
698 return -errno;
699 }
700 ptr = data + aligned_diff;
701
702 ret = out->sparse_ops->write_data_chunk(out, len, ptr);
703
704 munmap(data, buffer_size);
705
706 return ret;
707 }
708
709 /* Write a contiguous region of data blocks from a file */
710 int write_file_chunk(struct output_file *out, unsigned int len,
711 const char *file, int64_t offset)
712 {
713 int ret;
714
715 int file_fd = open(file, O_RDONLY);
716 if (file_fd < 0) {
717 return -errno;
718 }
719
720 ret = write_fd_chunk(out, len, file_fd, offset);
721
722 close(file_fd);
723
724 return ret;
725 }
726
727 int write_skip_chunk(struct output_file *out, int64_t len)
728 {
729 return out->sparse_ops->write_skip_chunk(out, len);
730 }