tools/mtd-utils/patches/136-mkfs.ubifs-xz-support.patch

   1 --- a/Makefile
   2 +++ b/Makefile
   3 @@ -4,7 +4,7 @@
   4  VERSION = 1.5.1
   5
   6  CPPFLAGS += -D_GNU_SOURCE -I./include -I$(BUILDDIR)/include -I./ubi-utils/include $(ZLIBCPPFLAGS) $(LZOCPPFLAGS) $(UUIDCPPFLAGS)
   7 -CPPFLAGS += -I./include/linux/lzma
   8 +CPPFLAGS += $(XZCPPFLAGS) -I./include/linux/lzma
   9
  10  ifeq ($(WITHOUT_XATTR), 1)
  11    CPPFLAGS += -DWITHOUT_XATTR
  12 @@ -113,8 +113,13 @@ ifeq ($(WITHOUT_LZO), 1)
  13  else
  14    LZOLDLIBS = -llzo2
  15  endif
  16 +ifeq ($(WITHOUT_XZ), 1)
  17 +  CPPFLAGS += -DWITHOUT_XZ
  18 +else
  19 +  XZLDLIBS = -llzma
  20 +endif
  21
  22 -LDLIBS_mkfs.ubifs = -lz $(LZOLDLIBS) -lm -luuid
  23 +LDLIBS_mkfs.ubifs = -lz $(LZOLDLIBS) $(XZLDLIBS) -lm -luuid
  24  $(call mkdep,mkfs.ubifs/,mkfs.ubifs,,ubi-utils/libubi.a)
  25
  26  #
  27 --- a/mkfs.ubifs/compr.c
  28 +++ b/mkfs.ubifs/compr.c
  29 @@ -126,6 +126,114 @@ static inline int lzo_init(void) { retur
  30  static inline void lzo_fini(void) { }
  31  #endif
  32
  33 +#ifndef WITHOUT_XZ
  34 +
  35 +#include <lzma.h>
  36 +
  37 +struct xz_ctx {
  38 +       lzma_filter     filters[3];
  39 +       lzma_options_lzma opts;
  40 +};
  41 +
  42 +static struct xz_ctx *xz_ctx;
  43 +
  44 +#define LZMA_COMPRESSION_LEVEL 9
  45 +
  46 +static struct xz_ctx *xz_ctx_init(void)
  47 +{
  48 +       struct xz_ctx *ctx;
  49 +       lzma_options_lzma *opts_lzma;
  50 +       uint32_t preset;
  51 +       int ret;
  52 +
  53 +       ctx = malloc(sizeof(struct xz_ctx));
  54 +       if (ctx == NULL)
  55 +               goto err;
  56 +
  57 +       memset(ctx, 0, sizeof(struct xz_ctx));
  58 +
  59 +       opts_lzma = &ctx->opts;
  60 +
  61 +       preset = LZMA_COMPRESSION_LEVEL | LZMA_PRESET_EXTREME;
  62 +       ret = lzma_lzma_preset(opts_lzma, preset);
  63 +       if (ret)
  64 +               goto err_free_ctx;
  65 +
  66 +       /* TODO: allow to specify LZMA options via command line */
  67 +#if 0
  68 +       opts_lzma->lc = 3;
  69 +       opts_lzma->lp = 0;
  70 +       opts_lzma->pb = 2;
  71 +       opts_lzma->nice_len = 64;
  72 +#else
  73 +       opts_lzma->lc = 0;
  74 +       opts_lzma->lp = 2;
  75 +       opts_lzma->pb = 2;
  76 +       opts_lzma->nice_len = 64;
  77 +#endif
  78 +
  79 +       ctx->filters[0].id = LZMA_FILTER_LZMA2;
  80 +       ctx->filters[0].options = opts_lzma;
  81 +       ctx->filters[1].id = LZMA_VLI_UNKNOWN;
  82 +
  83 +       return ctx;
  84 +
  85 +err_free_ctx:
  86 +       free(ctx);
  87 +err:
  88 +       return NULL;
  89 +}
  90 +
  91 +static void xz_ctx_free(struct xz_ctx *ctx)
  92 +{
  93 +       free(ctx);
  94 +}
  95 +
  96 +static int xz_init(void)
  97 +{
  98 +       xz_ctx = xz_ctx_init();
  99 +       if (xz_ctx == NULL)
 100 +               return -1;
 101 +
 102 +       return 0;
 103 +}
 104 +
 105 +static void xz_fini(void)
 106 +{
 107 +       xz_ctx_free(xz_ctx);
 108 +}
 109 +
 110 +static int xz_compress(void *in_buf, size_t in_len, void *out_buf,
 111 +                      size_t *out_len)
 112 +{
 113 +       size_t ret_len;
 114 +       lzma_ret ret_xz;
 115 +       int ret;
 116 +
 117 +       ret = -1;
 118 +
 119 +       ret_len = 0;
 120 +       ret_xz = lzma_stream_buffer_encode(xz_ctx->filters, LZMA_CHECK_CRC32,
 121 +                                          NULL, in_buf, in_len, out_buf,
 122 +                                          &ret_len, *out_len);
 123 +       if (ret_xz != LZMA_OK) {
 124 +               fprintf(stderr, "XZ error: %d\n", (int) ret_xz);
 125 +               goto out;
 126 +       }
 127 +
 128 +       *out_len = ret_len;
 129 +
 130 +       ret = 0;
 131 +out:
 132 +       return ret;
 133 +}
 134 +#else
 135 +static inline int xz_init(void) { return 0; }
 136 +static inline void xz_fini(void) { }
 137 +static inline int xz_compress(void *in_buf, size_t in_len, void *out_buf,
 138 +                             size_t *out_len) { return -1; }
 139 +#endif
 140 +
 141  static int no_compress(void *in_buf, size_t in_len, void *out_buf,
 142                        size_t *out_len)
 143  {
 144 @@ -198,6 +306,9 @@ int compress_data(void *in_buf, size_t i
 145                 case MKFS_UBIFS_COMPR_LZO:
 146                         ret = lzo_compress(in_buf, in_len, out_buf, out_len);
 147                         break;
 148 +               case MKFS_UBIFS_COMPR_XZ:
 149 +                       ret = xz_compress(in_buf, in_len, out_buf, out_len);
 150 +                       break;
 151                 case MKFS_UBIFS_COMPR_ZLIB:
 152                         ret = zlib_deflate(in_buf, in_len, out_buf, out_len);
 153                         break;
 154 @@ -225,12 +336,18 @@ int init_compression(void)
 155         if (ret)
 156                 goto err;
 157
 158 +       ret = xz_init();
 159 +       if (ret)
 160 +               goto err_lzo;
 161 +
 162         zlib_buf = malloc(UBIFS_BLOCK_SIZE * WORST_COMPR_FACTOR);
 163         if (!zlib_buf)
 164 -               goto err_lzo;
 165 +               goto err_xz;
 166
 167         return 0;
 168
 169 +err_xz:
 170 +       xz_fini();
 171  err_lzo:
 172         lzo_fini();
 173  err:
 174 @@ -240,6 +357,7 @@ err:
 175  void destroy_compression(void)
 176  {
 177         free(zlib_buf);
 178 +       xz_fini();
 179         lzo_fini();
 180         if (errcnt)
 181                 fprintf(stderr, "%llu compression errors occurred\n", errcnt);
 182 --- a/mkfs.ubifs/compr.h
 183 +++ b/mkfs.ubifs/compr.h
 184 @@ -36,6 +36,7 @@ enum compression_type
 185         MKFS_UBIFS_COMPR_NONE,
 186         MKFS_UBIFS_COMPR_LZO,
 187         MKFS_UBIFS_COMPR_ZLIB,
 188 +       MKFS_UBIFS_COMPR_XZ,
 189  };
 190
 191  int compress_data(void *in_buf, size_t in_len, void *out_buf, size_t *out_len,
 192 --- a/mkfs.ubifs/mkfs.ubifs.c
 193 +++ b/mkfs.ubifs/mkfs.ubifs.c
 194 @@ -99,6 +99,9 @@ struct ubifs_info info_;
 195  static struct ubifs_info *c = &info_;
 196  static libubi_t ubi;
 197
 198 +static int force_compr_set;
 199 +static int force_compr;
 200 +
 201  /* Debug levels are: 0 (none), 1 (statistics), 2 (files) ,3 (more details) */
 202  int debug_level;
 203  int verbose;
 204 @@ -133,7 +136,7 @@ static struct inum_mapping **hash_table;
 205  /* Inode creation sequence number */
 206  static unsigned long long creat_sqnum;
 207
 208 -static const char *optstring = "d:r:m:o:D:yh?vVe:c:g:f:Fp:k:x:X:j:R:l:j:UQq";
 209 +static const char *optstring = "d:r:m:o:D:yh?vVe:c:g:f:Fp:k:x:X:z:j:R:l:j:UQq";
 210
 211  static const struct option longopts[] = {
 212         {"root",               1, NULL, 'r'},
 213 @@ -151,6 +154,7 @@ static const struct option longopts[] =
 214         {"reserved",           1, NULL, 'R'},
 215         {"compr",              1, NULL, 'x'},
 216         {"favor-percent",      1, NULL, 'X'},
 217 +       {"force-compr",        1, NULL, 'z'},
 218         {"fanout",             1, NULL, 'f'},
 219         {"space-fixup",        0, NULL, 'F'},
 220         {"keyhash",            1, NULL, 'k'},
 221 @@ -178,11 +182,13 @@ static const char *helptext =
 222  "-o, --output=FILE        output to FILE\n"
 223  "-j, --jrn-size=SIZE      journal size\n"
 224  "-R, --reserved=SIZE      how much space should be reserved for the super-user\n"
 225 -"-x, --compr=TYPE         compression type - \"lzo\", \"favor_lzo\", \"zlib\" or\n"
 226 -"                         \"none\" (default: \"lzo\")\n"
 227 +"-x, --compr=TYPE         default compression type - \"lzo\", \"favor_lzo\",\n"
 228 +"                         \"zlib\" or \"none\" (default: \"lzo\")\n"
 229  "-X, --favor-percent      may only be used with favor LZO compression and defines\n"
 230  "                         how many percent better zlib should compress to make\n"
 231  "                         mkfs.ubifs use zlib instead of LZO (default 20%)\n"
 232 +"-z, --force-compr=TYPE   force to build the fs with different compression -\n"
 233 +"                         \"lzo\", \"zlib\" or \"none\"\n"
 234  "-f, --fanout=NUM         fanout NUM (default: 8)\n"
 235  "-F, --space-fixup        file-system free space has to be fixed up on first mount\n"
 236  "                         (requires kernel version 3.0 or greater)\n"
 237 @@ -472,6 +478,43 @@ static int open_ubi(const char *node)
 238         return 0;
 239  }
 240
 241 +static const char *get_compr_str(int compr)
 242 +{
 243 +       switch (compr) {
 244 +       case UBIFS_COMPR_LZO:
 245 +               return "lzo";
 246 +       case UBIFS_COMPR_ZLIB:
 247 +               return "zlib";
 248 +       case UBIFS_COMPR_XZ:
 249 +               return "xz";
 250 +       case UBIFS_COMPR_NONE:
 251 +               return "none";
 252 +       }
 253 +
 254 +       return "unknown";
 255 +}
 256 +
 257 +static int get_compr_option(char *opt, int *compr_type, int *favor_lzo)
 258 +{
 259 +       *compr_type = UBIFS_COMPR_LZO;
 260 +
 261 +       if (favor_lzo)
 262 +               *favor_lzo = 0;
 263 +
 264 +       if (favor_lzo && strcmp(optarg, "favor_lzo") == 0)
 265 +               *favor_lzo = 1;
 266 +       else if (strcmp(optarg, "zlib") == 0)
 267 +               *compr_type = UBIFS_COMPR_ZLIB;
 268 +       else if (strcmp(optarg, "xz") == 0)
 269 +               *compr_type = UBIFS_COMPR_XZ;
 270 +       else if (strcmp(optarg, "none") == 0)
 271 +               *compr_type = UBIFS_COMPR_NONE;
 272 +       else if (strcmp(optarg, "lzo") != 0)
 273 +               return -1;
 274 +
 275 +       return 0;
 276 +}
 277 +
 278  static int get_options(int argc, char**argv)
 279  {
 280         int opt, i;
 281 @@ -594,14 +637,13 @@ static int get_options(int argc, char**a
 282                                 return err_msg("bad key hash");
 283                         break;
 284                 case 'x':
 285 -                       if (strcmp(optarg, "favor_lzo") == 0)
 286 -                               c->favor_lzo = 1;
 287 -                       else if (strcmp(optarg, "zlib") == 0)
 288 -                               c->default_compr = UBIFS_COMPR_ZLIB;
 289 -                       else if (strcmp(optarg, "none") == 0)
 290 -                               c->default_compr = UBIFS_COMPR_NONE;
 291 -                       else if (strcmp(optarg, "lzo") != 0)
 292 -                               return err_msg("bad compressor name");
 293 +                       if (get_compr_option(optarg, &c->default_compr,
 294 +                                            &c->favor_lzo))
 295 +                               return err_msg("bad compressor name '%s'",
 296 +                                               optarg);
 297 +                       if (c->default_compr == UBIFS_COMPR_XZ)
 298 +                               return err_msg("'%s' can't be used as default compressor",
 299 +                                               optarg);
 300                         break;
 301                 case 'X':
 302                         c->favor_percent = strtol(optarg, &endp, 0);
 303 @@ -610,6 +652,12 @@ static int get_options(int argc, char**a
 304                                 return err_msg("bad favor LZO percent '%s'",
 305                                                optarg);
 306                         break;
 307 +               case 'z':
 308 +                       if (get_compr_option(optarg, &force_compr, NULL))
 309 +                               return err_msg("bad forced compressor name '%s'",
 310 +                                               optarg);
 311 +                       force_compr_set = 1;
 312 +                       break;
 313                 case 'j':
 314                         c->max_bud_bytes = get_bytes(optarg);
 315                         if (c->max_bud_bytes <= 0)
 316 @@ -684,6 +732,9 @@ static int get_options(int argc, char**a
 317                 c->min_io_size = 8;
 318         c->rp_size = add_space_overhead(c->rp_size);
 319
 320 +       if (force_compr_set == 0)
 321 +               force_compr = c->default_compr;
 322 +
 323         if (verbose) {
 324                 printf("mkfs.ubifs\n");
 325                 printf("\troot:         %s\n", root);
 326 @@ -693,17 +744,10 @@ static int get_options(int argc, char**a
 327                 printf("\toutput:       %s\n", output);
 328                 printf("\tjrn_size:     %llu\n", c->max_bud_bytes);
 329                 printf("\treserved:     %llu\n", c->rp_size);
 330 -               switch (c->default_compr) {
 331 -               case UBIFS_COMPR_LZO:
 332 -                       printf("\tcompr:        lzo\n");
 333 -                       break;
 334 -               case UBIFS_COMPR_ZLIB:
 335 -                       printf("\tcompr:        zlib\n");
 336 -                       break;
 337 -               case UBIFS_COMPR_NONE:
 338 -                       printf("\tcompr:        none\n");
 339 -                       break;
 340 -               }
 341 +               printf("\tcompr:        %s\n", get_compr_str(c->default_compr));
 342 +               if (force_compr_set)
 343 +                       printf("\tforced compr: %s\n",
 344 +                              get_compr_str(force_compr));
 345                 printf("\tkeyhash:      %s\n", (c->key_hash == key_r5_hash) ?
 346                                                 "r5" : "test");
 347                 printf("\tfanout:       %d\n", c->fanout);
 348 @@ -1284,7 +1328,7 @@ static int add_file(const char *path_nam
 349                         use_compr = UBIFS_COMPR_LZO;
 350                 else
 351  #endif
 352 -                       use_compr = c->default_compr;
 353 +                       use_compr = force_compr;
 354                 compr_type = compress_data(buf, bytes_read, &dn->data,
 355                                            &out_len, use_compr);
 356                 dn->compr_type = cpu_to_le16(compr_type);
 357 --- a/mkfs.ubifs/mkfs.ubifs.h
 358 +++ b/mkfs.ubifs/mkfs.ubifs.h
 359 @@ -83,6 +83,9 @@
 360  #if MKFS_UBIFS_COMPR_ZLIB != UBIFS_COMPR_ZLIB
 361  #error MKFS_UBIFS_COMPR_ZLIB != UBIFS_COMPR_ZLIB
 362  #endif
 363 +#if MKFS_UBIFS_COMPR_XZ != UBIFS_COMPR_XZ
 364 +#error MKFS_UBIFS_COMPR_XZ != UBIFS_COMPR_XZ
 365 +#endif
 366
 367  extern int verbose;
 368  extern int debug_level;
 369 --- a/include/mtd/ubifs-media.h
 370 +++ b/include/mtd/ubifs-media.h
 371 @@ -313,6 +313,7 @@ enum {
 372         UBIFS_COMPR_NONE,
 373         UBIFS_COMPR_LZO,
 374         UBIFS_COMPR_ZLIB,
 375 +       UBIFS_COMPR_XZ,
 376         UBIFS_COMPR_TYPES_CNT,
 377  };
 378