tools/mtd-utils/patches/136-mkfs.ubifs-xz-support.patch

   1 --- a/Makefile
   2 +++ b/Makefile
   3 @@ -1,7 +1,7 @@
   4
   5  # -*- sh -*-
   6
   7 -CPPFLAGS += -I./include $(ZLIBCPPFLAGS) $(LZOCPPFLAGS) -I./include/linux/lzma
   8 +CPPFLAGS += -I./include $(ZLIBCPPFLAGS) $(LZOCPPFLAGS) $(XZCPPFLAGS) -I./include/linux/lzma
   9
  10  ifeq ($(WITHOUT_XATTR), 1)
  11    CPPFLAGS += -DWITHOUT_XATTR
  12 --- a/mkfs.ubifs/compr.c
  13 +++ b/mkfs.ubifs/compr.c
  14 @@ -127,6 +127,114 @@ static inline int lzo_init(void) { retur
  15  static inline void lzo_fini(void) { }
  16  #endif
  17
  18 +#ifndef WITHOUT_XZ
  19 +
  20 +#include <lzma.h>
  21 +
  22 +struct xz_ctx {
  23 +       lzma_filter     filters[3];
  24 +       lzma_options_lzma opts;
  25 +};
  26 +
  27 +static struct xz_ctx *xz_ctx;
  28 +
  29 +#define LZMA_COMPRESSION_LEVEL 9
  30 +
  31 +static struct xz_ctx *xz_ctx_init(void)
  32 +{
  33 +       struct xz_ctx *ctx;
  34 +       lzma_options_lzma *opts_lzma;
  35 +       uint32_t preset;
  36 +       int ret;
  37 +
  38 +       ctx = malloc(sizeof(struct xz_ctx));
  39 +       if (ctx == NULL)
  40 +               goto err;
  41 +
  42 +       memset(ctx, 0, sizeof(struct xz_ctx));
  43 +
  44 +       opts_lzma = &ctx->opts;
  45 +
  46 +       preset = LZMA_COMPRESSION_LEVEL | LZMA_PRESET_EXTREME;
  47 +       ret = lzma_lzma_preset(opts_lzma, preset);
  48 +       if (ret)
  49 +               goto err_free_ctx;
  50 +
  51 +       /* TODO: allow to specify LZMA options via command line */
  52 +#if 0
  53 +       opts_lzma->lc = 3;
  54 +       opts_lzma->lp = 0;
  55 +       opts_lzma->pb = 2;
  56 +       opts_lzma->nice_len = 64;
  57 +#else
  58 +       opts_lzma->lc = 0;
  59 +       opts_lzma->lp = 2;
  60 +       opts_lzma->pb = 2;
  61 +       opts_lzma->nice_len = 64;
  62 +#endif
  63 +
  64 +       ctx->filters[0].id = LZMA_FILTER_LZMA2;
  65 +       ctx->filters[0].options = opts_lzma;
  66 +       ctx->filters[1].id = LZMA_VLI_UNKNOWN;
  67 +
  68 +       return ctx;
  69 +
  70 +err_free_ctx:
  71 +       free(ctx);
  72 +err:
  73 +       return NULL;
  74 +}
  75 +
  76 +static void xz_ctx_free(struct xz_ctx *ctx)
  77 +{
  78 +       free(ctx);
  79 +}
  80 +
  81 +static int xz_init(void)
  82 +{
  83 +       xz_ctx = xz_ctx_init();
  84 +       if (xz_ctx == NULL)
  85 +               return -1;
  86 +
  87 +       return 0;
  88 +}
  89 +
  90 +static void xz_fini(void)
  91 +{
  92 +       xz_ctx_free(xz_ctx);
  93 +}
  94 +
  95 +static int xz_compress(void *in_buf, size_t in_len, void *out_buf,
  96 +                      size_t *out_len)
  97 +{
  98 +       size_t ret_len;
  99 +       lzma_ret ret_xz;
 100 +       int ret;
 101 +
 102 +       ret = -1;
 103 +
 104 +       ret_len = 0;
 105 +       ret_xz = lzma_stream_buffer_encode(xz_ctx->filters, LZMA_CHECK_CRC32,
 106 +                                          NULL, in_buf, in_len, out_buf,
 107 +                                          &ret_len, *out_len);
 108 +       if (ret_xz != LZMA_OK) {
 109 +               fprintf(stderr, "XZ error: %d\n", (int) ret_xz);
 110 +               goto out;
 111 +       }
 112 +
 113 +       *out_len = ret_len;
 114 +
 115 +       ret = 0;
 116 +out:
 117 +       return ret;
 118 +}
 119 +#else
 120 +static inline int xz_init(void) { return 0; }
 121 +static inline void xz_fini(void) { }
 122 +static inline int xz_compress(void *in_buf, size_t in_len, void *out_buf,
 123 +                             size_t *out_len) { return -1; }
 124 +#endif
 125 +
 126  static int no_compress(void *in_buf, size_t in_len, void *out_buf,
 127                        size_t *out_len)
 128  {
 129 @@ -199,6 +307,9 @@ int compress_data(void *in_buf, size_t i
 130                 case MKFS_UBIFS_COMPR_LZO:
 131                         ret = lzo_compress(in_buf, in_len, out_buf, out_len);
 132                         break;
 133 +               case MKFS_UBIFS_COMPR_XZ:
 134 +                       ret = xz_compress(in_buf, in_len, out_buf, out_len);
 135 +                       break;
 136                 case MKFS_UBIFS_COMPR_ZLIB:
 137                         ret = zlib_deflate(in_buf, in_len, out_buf, out_len);
 138                         break;
 139 @@ -226,12 +337,18 @@ int init_compression(void)
 140         if (ret)
 141                 goto err;
 142
 143 +       ret = xz_init();
 144 +       if (ret)
 145 +               goto err_lzo;
 146 +
 147         zlib_buf = malloc(UBIFS_BLOCK_SIZE * WORST_COMPR_FACTOR);
 148         if (!zlib_buf)
 149 -               goto err_lzo;
 150 +               goto err_xz;
 151
 152         return 0;
 153
 154 +err_xz:
 155 +       xz_fini();
 156  err_lzo:
 157         lzo_fini();
 158  err:
 159 @@ -241,6 +358,7 @@ err:
 160  void destroy_compression(void)
 161  {
 162         free(zlib_buf);
 163 +       xz_fini();
 164         lzo_fini();
 165         if (errcnt)
 166                 fprintf(stderr, "%llu compression errors occurred\n", errcnt);
 167 --- a/mkfs.ubifs/compr.h
 168 +++ b/mkfs.ubifs/compr.h
 169 @@ -36,6 +36,7 @@ enum compression_type
 170         MKFS_UBIFS_COMPR_NONE,
 171         MKFS_UBIFS_COMPR_LZO,
 172         MKFS_UBIFS_COMPR_ZLIB,
 173 +       MKFS_UBIFS_COMPR_XZ,
 174  };
 175
 176  int compress_data(void *in_buf, size_t in_len, void *out_buf, size_t *out_len,
 177 --- a/mkfs.ubifs/Makefile
 178 +++ b/mkfs.ubifs/Makefile
 179 @@ -6,21 +6,33 @@ ALL_SOURCES=*.[ch] hashtable/*.[ch]
 180
 181  TARGETS = mkfs.ubifs
 182
 183 +MKFS_UBIFS_OBJS = $(addprefix $(BUILDDIR)/,\
 184 +       crc16.o lpt.o compr.o devtable.o \
 185 +       hashtable/hashtable.o hashtable/hashtable_itr.o)
 186 +
 187  ifeq ($(WITHOUT_LZO), 1)
 188    CPPFLAGS += -DWITHOUT_LZO
 189  else
 190    LZOLDLIBS = -llzo2
 191  endif
 192
 193 -LDLIBS_mkfs.ubifs = -lz $(LZOLDLIBS) -lm -luuid -L$(BUILDDIR)/../ubi-utils/ -lubi
 194 +ifeq ($(WITHOUT_XZ), 1)
 195 +  CPPFLAGS += -DWITHOUT_XZ
 196 +else
 197 +ifneq ($(LZMA_STATIC_LIB),)
 198 +  MKFS_UBIFS_OBJS += $(LZMA_STATIC_LIB)
 199 +else
 200 +  XZLDLIBS = -llzma
 201 +endif
 202 +endif
 203 +
 204 +LDLIBS_mkfs.ubifs = -lz $(LZOLDLIBS) $(XZLDLIBS) -lm -luuid -L$(BUILDDIR)/../ubi-utils/ -lubi
 205  LDLIBS_mkfs.ubifs += -L$(BUILDDIR)/../lib -lmtd
 206 -LDLIBS_mkfs.ubifs += $(ZLIBLDFLAGS) $(LZOLDFLAGS)
 207 +LDLIBS_mkfs.ubifs += $(ZLIBLDFLAGS) $(LZOLDFLAGS) $(XZLDFLAGS)
 208
 209  include ../common.mk
 210
 211 -$(BUILDDIR)/mkfs.ubifs: $(addprefix $(BUILDDIR)/,\
 212 -       crc16.o lpt.o compr.o devtable.o \
 213 -       hashtable/hashtable.o hashtable/hashtable_itr.o)
 214 +$(BUILDDIR)/mkfs.ubifs: $(MKFS_UBIFS_OBJS)
 215
 216  clean::
 217         rm -f $(BUILDDIR)/hashtable/*.o cscope.*
 218 --- a/mkfs.ubifs/mkfs.ubifs.c
 219 +++ b/mkfs.ubifs/mkfs.ubifs.c
 220 @@ -98,6 +98,9 @@ struct ubifs_info info_;
 221  static struct ubifs_info *c = &info_;
 222  static libubi_t ubi;
 223
 224 +static int force_compr_set;
 225 +static int force_compr;
 226 +
 227  /* Debug levels are: 0 (none), 1 (statistics), 2 (files) ,3 (more details) */
 228  int debug_level;
 229  int verbose;
 230 @@ -132,7 +135,7 @@ static struct inum_mapping **hash_table;
 231  /* Inode creation sequence number */
 232  static unsigned long long creat_sqnum;
 233
 234 -static const char *optstring = "d:r:m:o:D:h?vVe:c:g:f:Fp:k:x:X:j:R:l:j:UQq";
 235 +static const char *optstring = "d:r:m:o:D:h?vVe:c:g:f:Fp:k:x:X:y:j:R:l:j:UQq";
 236
 237  static const struct option longopts[] = {
 238         {"root",               1, NULL, 'r'},
 239 @@ -149,6 +152,7 @@ static const struct option longopts[] =
 240         {"reserved",           1, NULL, 'R'},
 241         {"compr",              1, NULL, 'x'},
 242         {"favor-percent",      1, NULL, 'X'},
 243 +       {"force-compr",        1, NULL, 'y'},
 244         {"fanout",             1, NULL, 'f'},
 245         {"space-fixup",        0, NULL, 'F'},
 246         {"keyhash",            1, NULL, 'k'},
 247 @@ -178,11 +182,13 @@ static const char *helptext =
 248  "-o, --output=FILE        output to FILE\n"
 249  "-j, --jrn-size=SIZE      journal size\n"
 250  "-R, --reserved=SIZE      how much space should be reserved for the super-user\n"
 251 -"-x, --compr=TYPE         compression type - \"lzo\", \"favor_lzo\", \"zlib\" or\n"
 252 -"                         \"none\" (default: \"lzo\")\n"
 253 +"-x, --compr=TYPE         default compression type - \"lzo\", \"favor_lzo\",\n"
 254 +"                         \"zlib\" or \"none\" (default: \"lzo\")\n"
 255  "-X, --favor-percent      may only be used with favor LZO compression and defines\n"
 256  "                         how many percent better zlib should compress to make\n"
 257  "                         mkfs.ubifs use zlib instead of LZO (default 20%)\n"
 258 +"-y, --force-compr=TYPE   force to build the fs with different compression -\n"
 259 +"                         \"lzo\", \"zlib\" or \"none\"\n"
 260  "-f, --fanout=NUM         fanout NUM (default: 8)\n"
 261  "-F, --space-fixup        file-system free space has to be fixed up on first mount\n"
 262  "                         (requires kernel version 3.0 or greater)\n"
 263 @@ -530,6 +536,43 @@ static int open_ubi(const char *node)
 264         return 0;
 265  }
 266
 267 +static const char *get_compr_str(int compr)
 268 +{
 269 +       switch (compr) {
 270 +       case UBIFS_COMPR_LZO:
 271 +               return "lzo";
 272 +       case UBIFS_COMPR_ZLIB:
 273 +               return "zlib";
 274 +       case UBIFS_COMPR_XZ:
 275 +               return "xz";
 276 +       case UBIFS_COMPR_NONE:
 277 +               return "none";
 278 +       }
 279 +
 280 +       return "unknown";
 281 +}
 282 +
 283 +static int get_compr_option(char *opt, int *compr_type, int *favor_lzo)
 284 +{
 285 +       *compr_type = UBIFS_COMPR_LZO;
 286 +
 287 +       if (favor_lzo)
 288 +               *favor_lzo = 0;
 289 +
 290 +       if (favor_lzo && strcmp(optarg, "favor_lzo") == 0)
 291 +               *favor_lzo = 1;
 292 +       else if (strcmp(optarg, "zlib") == 0)
 293 +               *compr_type = UBIFS_COMPR_ZLIB;
 294 +       else if (strcmp(optarg, "xz") == 0)
 295 +               *compr_type = UBIFS_COMPR_XZ;
 296 +       else if (strcmp(optarg, "none") == 0)
 297 +               *compr_type = UBIFS_COMPR_NONE;
 298 +       else if (strcmp(optarg, "lzo") != 0)
 299 +               return -1;
 300 +
 301 +       return 0;
 302 +}
 303 +
 304  static int get_options(int argc, char**argv)
 305  {
 306         int opt, i;
 307 @@ -649,14 +692,13 @@ static int get_options(int argc, char**a
 308                                 return err_msg("bad key hash");
 309                         break;
 310                 case 'x':
 311 -                       if (strcmp(optarg, "favor_lzo") == 0)
 312 -                               c->favor_lzo = 1;
 313 -                       else if (strcmp(optarg, "zlib") == 0)
 314 -                               c->default_compr = UBIFS_COMPR_ZLIB;
 315 -                       else if (strcmp(optarg, "none") == 0)
 316 -                               c->default_compr = UBIFS_COMPR_NONE;
 317 -                       else if (strcmp(optarg, "lzo") != 0)
 318 -                               return err_msg("bad compressor name");
 319 +                       if (get_compr_option(optarg, &c->default_compr,
 320 +                                            &c->favor_lzo))
 321 +                               return err_msg("bad compressor name '%s'",
 322 +                                               optarg);
 323 +                       if (c->default_compr == UBIFS_COMPR_XZ)
 324 +                               return err_msg("'%s' can't be used as default compressor",
 325 +                                               optarg);
 326                         break;
 327                 case 'X':
 328                         c->favor_percent = strtol(optarg, &endp, 0);
 329 @@ -665,6 +707,12 @@ static int get_options(int argc, char**a
 330                                 return err_msg("bad favor LZO percent '%s'",
 331                                                optarg);
 332                         break;
 333 +               case 'y':
 334 +                       if (get_compr_option(optarg, &force_compr, NULL))
 335 +                               return err_msg("bad forced compressor name '%s'",
 336 +                                               optarg);
 337 +                       force_compr_set = 1;
 338 +                       break;
 339                 case 'j':
 340                         c->max_bud_bytes = get_bytes(optarg);
 341                         if (c->max_bud_bytes <= 0)
 342 @@ -749,6 +797,9 @@ static int get_options(int argc, char**a
 343                 c->min_io_size = 8;
 344         c->rp_size = add_space_overhead(c->rp_size);
 345
 346 +       if (force_compr_set == 0)
 347 +               force_compr = c->default_compr;
 348 +
 349         if (verbose) {
 350                 printf("mkfs.ubifs\n");
 351                 printf("\troot:         %s\n", root);
 352 @@ -758,17 +809,10 @@ static int get_options(int argc, char**a
 353                 printf("\toutput:       %s\n", output);
 354                 printf("\tjrn_size:     %llu\n", c->max_bud_bytes);
 355                 printf("\treserved:     %llu\n", c->rp_size);
 356 -               switch (c->default_compr) {
 357 -               case UBIFS_COMPR_LZO:
 358 -                       printf("\tcompr:        lzo\n");
 359 -                       break;
 360 -               case UBIFS_COMPR_ZLIB:
 361 -                       printf("\tcompr:        zlib\n");
 362 -                       break;
 363 -               case UBIFS_COMPR_NONE:
 364 -                       printf("\tcompr:        none\n");
 365 -                       break;
 366 -               }
 367 +               printf("\tcompr:        %s\n", get_compr_str(c->default_compr));
 368 +               if (force_compr_set)
 369 +                       printf("\tforced compr: %s\n",
 370 +                              get_compr_str(force_compr));
 371                 printf("\tkeyhash:      %s\n", (c->key_hash == key_r5_hash) ?
 372                                                 "r5" : "test");
 373                 printf("\tfanout:       %d\n", c->fanout);
 374 @@ -1353,7 +1397,7 @@ static int add_file(const char *path_nam
 375                         use_compr = UBIFS_COMPR_LZO;
 376                 else
 377  #endif
 378 -                       use_compr = c->default_compr;
 379 +                       use_compr = force_compr;
 380                 compr_type = compress_data(buf, bytes_read, &dn->data,
 381                                            &out_len, use_compr);
 382                 dn->compr_type = cpu_to_le16(compr_type);
 383 --- a/mkfs.ubifs/mkfs.ubifs.h
 384 +++ b/mkfs.ubifs/mkfs.ubifs.h
 385 @@ -77,6 +77,9 @@
 386  #if MKFS_UBIFS_COMPR_ZLIB != UBIFS_COMPR_ZLIB
 387  #error MKFS_UBIFS_COMPR_ZLIB != UBIFS_COMPR_ZLIB
 388  #endif
 389 +#if MKFS_UBIFS_COMPR_XZ != UBIFS_COMPR_XZ
 390 +#error MKFS_UBIFS_COMPR_XZ != UBIFS_COMPR_XZ
 391 +#endif
 392
 393  extern int verbose;
 394  extern int debug_level;
 395 --- a/mkfs.ubifs/ubifs-media.h
 396 +++ b/mkfs.ubifs/ubifs-media.h
 397 @@ -303,6 +303,7 @@ enum {
 398         UBIFS_COMPR_NONE,
 399         UBIFS_COMPR_LZO,
 400         UBIFS_COMPR_ZLIB,
 401 +       UBIFS_COMPR_XZ,
 402         UBIFS_COMPR_TYPES_CNT,
 403  };
 404