libopkg: store compressed package lists

To save storage space, store the package lists in compressed form and
decompress them on-the-fly during parsing.

Signed-off-by: Jo-Philipp Wich <jo@mein.io>

diff --git a/libopkg/opkg.c b/libopkg/opkg.c
index 7c3e18f878e5fe05910cfd255f596268d11fb431..dbb82fbb1c2c4b74045368d9153a1fa054c6c8d3 100644 (file)
--- a/libopkg/opkg.c
+++ b/libopkg/opkg.c
@@ -592,49 +592,8 @@ opkg_update_package_lists(opkg_progress_callback_t progress_callback,
                                      src->gzip ? "Packages.gz" : "Packages");
 
                sprintf_alloc(&list_file_name, "%s/%s", lists_dir, src->name);
-               if (src->gzip) {
-                       FILE *in, *out;
-                       struct _curl_cb_data cb_data;
-                       char *tmp_file_name = NULL;
 
-                       sprintf_alloc(&tmp_file_name, "%s/%s.gz", tmp,
-                                     src->name);
-
-                       opkg_msg(INFO, "Downloading %s to %s...\n", url,
-                                       tmp_file_name);
-
-                       cb_data.cb = progress_callback;
-                       cb_data.progress_data = &pdata;
-                       cb_data.user_data = user_data;
-                       cb_data.start_range =
-                           100 * sources_done / sources_list_count;
-                       cb_data.finish_range =
-                           100 * (sources_done + 1) / sources_list_count;
-
-                       err = opkg_download(url, tmp_file_name,
-                                         (curl_progress_func) curl_progress_cb,
-                                         &cb_data, 0);
-
-                       if (err == 0) {
-                               opkg_msg(INFO, "Inflating %s...\n",
-                                               tmp_file_name);
-                               in = fopen(tmp_file_name, "r");
-                               out = fopen(list_file_name, "w");
-                               if (in && out)
-                                       unzip(in, out);
-                               else
-                                       err = 1;
-                               if (in)
-                                       fclose(in);
-                               if (out)
-                                       fclose(out);
-                               unlink(tmp_file_name);
-                       }
-                       free(tmp_file_name);
-               } else
-                       err = opkg_download(url, list_file_name, NULL, NULL, 0);
-
-               if (err) {
+               if (opkg_download(url, list_file_name, NULL, NULL, 0)) {
                        opkg_msg(ERROR, "Couldn't retrieve %s\n", url);
                        result = -1;
                }

diff --git a/libopkg/opkg_cmd.c b/libopkg/opkg_cmd.c
index 1a8f8573a290de2f85452ebda53cd4d0e6671c94..d1e91cbeae54bfdad58b0dfcedfcb63f79d0e065 100644 (file)
--- a/libopkg/opkg_cmd.c
+++ b/libopkg/opkg_cmd.c
@@ -162,30 +162,7 @@ opkg_update_cmd(int argc, char **argv)
              sprintf_alloc(&url, "%s/%s", src->value, src->gzip ? "Packages.gz" : "Packages");
 
          sprintf_alloc(&list_file_name, "%s/%s", lists_dir, src->name);
-         if (src->gzip) {
-             char *tmp_file_name;
-             FILE *in, *out;
-
-             sprintf_alloc (&tmp_file_name, "%s/%s.gz", tmp, src->name);
-             err = opkg_download(url, tmp_file_name, NULL, NULL, 0);
-             if (err == 0) {
-                  opkg_msg(NOTICE, "Inflating %s.\n", url);
-                  in = fopen (tmp_file_name, "r");
-                  out = fopen (list_file_name, "w");
-                  if (in && out)
-                       unzip (in, out);
-                  else
-                       err = 1;
-                  if (in)
-                       fclose (in);
-                  if (out)
-                       fclose (out);
-                  unlink (tmp_file_name);
-             }
-             free(tmp_file_name);
-         } else
-             err = opkg_download(url, list_file_name, NULL, NULL, 0);
-         if (err) {
+         if (opkg_download(url, list_file_name, NULL, NULL, 0)) {
               failures++;
          } else {
               opkg_msg(NOTICE, "Updated list of available packages in %s.\n",

diff --git a/libopkg/pkg_hash.c b/libopkg/pkg_hash.c
index 7f76da130d0c5f20be22e4fe203c1a2574ad99bc..5f9a7062eb8c706c2b940a6ce45e911cd07e398d 100644 (file)
--- a/libopkg/pkg_hash.c
+++ b/libopkg/pkg_hash.c
@@ -29,6 +29,7 @@
 #include "sprintf_alloc.h"
 #include "file_util.h"
 #include "libbb/libbb.h"
+#include "libbb/gzip.h"
 
 void
 pkg_hash_init(void)
@@ -106,8 +107,15 @@ pkg_hash_add_from_file(const char *file_name,
        char *buf;
        const size_t len = 4096;
        int ret = 0;
+       struct gzip_handle zh;
+
+       if (src && src->gzip) {
+               fp = gzip_fdopen(&zh, file_name);
+       }
+       else {
+               fp = fopen(file_name, "r");
+       }
 
-       fp = fopen(file_name, "r");
        if (fp == NULL) {
                opkg_perror(ERROR, "Failed to open %s", file_name);
                return -1;
@@ -155,6 +163,9 @@ pkg_hash_add_from_file(const char *file_name,
        free(buf);
        fclose(fp);
 
+       if (src && src->gzip)
+               gzip_close(&zh);
+
        return ret;
 }