libopkg: store checksums in binary form, use integer index for architecture

Instead of storing a copy of the architecture string and architecture
priority value in each pkg_t instance, declare a 3 bit wide field which
allows referencing the architecture in the global config array by index.

The 3 bit field allows referencing up to 8 different architectures which
is more than enough for the systems we target with opkg. Another nice side
effect is that we can coalesce this field with other flag values in pkg_t,
saving 4 bytes for an int member.

Also convert the hexadecimal checksums to binary format before storing them in
pkg_t's blob buffer to save 50% of the space per checksum.

Signed-off-by: Jo-Philipp Wich <jo@mein.io>

diff --git a/libopkg/file_util.c b/libopkg/file_util.c
index 4f949acf730a4785899e3f0b40e2055c009903f2..912b147ad306766f6275e93a3b9860de81b29242 100644 (file)
--- a/libopkg/file_util.c
+++ b/libopkg/file_util.c
@@ -23,6 +23,7 @@
 #include <sys/stat.h>
 #include <dirent.h>
 #include <unistd.h>
+#include <ctype.h>
 
 #include "sprintf_alloc.h"
 #include "file_util.h"
@@ -228,6 +229,57 @@ char *file_sha256sum_alloc(const char *file_name)
 
 #endif
 
+char *checksum_bin2hex(const char *src, size_t len)
+{
+       char *p;
+       static char buf[65];
+       static const unsigned char bin2hex[16] = {
+               '0', '1', '2', '3',
+               '4', '5', '6', '7',
+               '8', '9', 'a', 'b',
+               'c', 'd', 'e', 'f'
+       };
+
+       if (len > 32)
+               return NULL;
+
+       for (p = buf; len > 0; src++, len--) {
+               *p++ = bin2hex[*src / 16];
+               *p++ = bin2hex[*src % 16];
+       }
+
+       *p = 0;
+
+       return buf;
+}
+
+char *checksum_hex2bin(const char *src, size_t *len)
+{
+       char *p;
+       size_t slen;
+       static char buf[32];
+
+       while (isspace(*src))
+               src++;
+
+       slen = strlen(src);
+
+       if (slen > 64) {
+               *len = 0;
+               return NULL;
+       }
+
+#define hex(c) \
+       (c >= 'a' ? (c - 'a') : (c >= 'A' ? (c - 'A') : (c - '0')))
+
+       for (p = buf, *len = 0;
+            slen > 0 && isxdigit(src[0]) && isxdigit(src[1]);
+            slen--, src += 2, (*len)++)
+               *p++ = hex(src[0]) * 16 + hex(src[1]);
+
+       return buf;
+}
+
 int rm_r(const char *path)
 {
        int ret = 0;

diff --git a/libopkg/file_util.h b/libopkg/file_util.h
index a7e2a3812d873c4efc8f76da9203dcc4245ec245..d8e676790aaa16c8917920fadacef4bdc781a56d 100644 (file)
--- a/libopkg/file_util.h
+++ b/libopkg/file_util.h
@@ -28,4 +28,7 @@ char *file_md5sum_alloc(const char *file_name);
 char *file_sha256sum_alloc(const char *file_name);
 int rm_r(const char *path);
 
+char *checksum_bin2hex(const char *src, size_t len);
+char *checksum_hex2bin(const char *src, size_t *len);
+
 #endif

diff --git a/libopkg/opkg.c b/libopkg/opkg.c
index 9e278cd7af621a1b47eb90fc73a96f146c2aa8e4..d8c17cdf87bd099adaa1e10c5cd528d14059de12 100644 (file)
--- a/libopkg/opkg.c
+++ b/libopkg/opkg.c
@@ -749,7 +749,7 @@ pkg_t *opkg_find_package(const char *name, const char *ver, const char *arch,
 
                /* check architecture */
                if (arch) {
-                       if (sstrcmp(pkg_get_string(pkg, PKG_ARCHITECTURE), arch))
+                       if (sstrcmp(pkg_get_architecture(pkg), arch))
                                continue;
                }
 

diff --git a/libopkg/opkg_install.c b/libopkg/opkg_install.c
index 55c124b3797f4c0c9737d5fa33628e7b0ab3760d..85159bf207a15cb0e7bc7a883528bf2cb365c4dd 100644 (file)
--- a/libopkg/opkg_install.c
+++ b/libopkg/opkg_install.c
@@ -1285,7 +1285,7 @@ int opkg_install_pkg(pkg_t * pkg, int from_upgrade)
        if (!pkg_arch_supported(pkg)) {
                opkg_msg(ERROR,
                         "INTERNAL ERROR: architecture %s for pkg %s is unsupported.\n",
-                        pkg_get_string(pkg, PKG_ARCHITECTURE), pkg->name);
+                        pkg_get_architecture(pkg), pkg->name);
                return -1;
        }
        if (pkg->state_status == SS_INSTALLED && conf->nodeps == 0) {

diff --git a/libopkg/pkg.c b/libopkg/pkg.c
index a1ee5a2ba8121537e4885f9011f6bf57dba0fefd..0731bd0f1972fea8ff290e7b5c7e835c9c8bfbab 100644 (file)
--- a/libopkg/pkg.c
+++ b/libopkg/pkg.c
@@ -85,6 +85,8 @@ static void pkg_init(pkg_t * pkg)
        pkg->essential = 0;
        pkg->provided_by_hand = 0;
 
+       pkg->arch_index = 0;
+
        blob_buf_init(&pkg->blob, 0);
 }
 
@@ -157,6 +159,59 @@ char *pkg_set_string(pkg_t *pkg, int id, const char *s)
        return p;
 }
 
+char *pkg_get_architecture(const pkg_t *pkg)
+{
+       nv_pair_list_elt_t *l;
+       int n = 1;
+
+       list_for_each_entry(l, &conf->arch_list.head, node) {
+               nv_pair_t *nv = (nv_pair_t *) l->data;
+               if (n++ == pkg->arch_index)
+                       return nv->name;
+       }
+
+       return NULL;
+}
+
+char *pkg_set_architecture(pkg_t *pkg, const char *architecture, ssize_t len)
+{
+       nv_pair_list_elt_t *l;
+       int n = 1;
+
+       list_for_each_entry(l, &conf->arch_list.head, node) {
+               nv_pair_t *nv = (nv_pair_t *) l->data;
+
+               if (!strncmp(nv->name, architecture, len) && nv->name[len] == '\0') {
+                       if (n >= 8) {
+                               opkg_msg(ERROR, "Internal error: too many different architectures\n");
+                               break;
+                       }
+
+                       pkg->arch_index = n;
+                       return nv->name;
+               }
+
+               n++;
+       }
+
+       pkg->arch_index = 0;
+       return NULL;
+}
+
+int pkg_get_arch_priority(const pkg_t *pkg)
+{
+       nv_pair_list_elt_t *l;
+       int n = 1;
+
+       list_for_each_entry(l, &conf->arch_list.head, node) {
+               nv_pair_t *nv = (nv_pair_t *) l->data;
+               if (n++ == pkg->arch_index)
+                       return strtol(nv->value, NULL, 0);
+       }
+
+       return 0;
+}
+
 
 static void compound_depend_deinit(compound_depend_t * depends)
 {
@@ -292,10 +347,8 @@ int pkg_merge(pkg_t * oldpkg, pkg_t * newpkg)
                oldpkg->src = newpkg->src;
        if (!oldpkg->dest)
                oldpkg->dest = newpkg->dest;
-       if (!pkg_get_string(oldpkg, PKG_ARCHITECTURE))
-               pkg_set_string(oldpkg, PKG_ARCHITECTURE, pkg_get_string(newpkg, PKG_ARCHITECTURE));
-       if (!pkg_get_int(oldpkg, PKG_ARCH_PRIORITY))
-               pkg_set_int(oldpkg, PKG_ARCH_PRIORITY, pkg_get_int(newpkg, PKG_ARCH_PRIORITY));
+       if (!oldpkg->arch_index)
+               oldpkg->arch_index = newpkg->arch_index;
        if (!pkg_get_string(oldpkg, PKG_SECTION))
                pkg_set_string(oldpkg, PKG_SECTION, pkg_get_string(newpkg, PKG_SECTION));
        if (!pkg_get_string(oldpkg, PKG_MAINTAINER))
@@ -544,7 +597,7 @@ void pkg_formatted_field(FILE * fp, pkg_t * pkg, const char *field)
        case 'a':
        case 'A':
                if (strcasecmp(field, "Architecture") == 0) {
-                       p = pkg_get_string(pkg, PKG_ARCHITECTURE);
+                       p = pkg_get_architecture(pkg);
                        if (p) {
                                fprintf(fp, "Architecture: %s\n",
                                        p);
@@ -983,8 +1036,8 @@ int pkg_name_version_and_architecture_compare(const void *p1, const void *p2)
        vercmp = pkg_compare_versions(a, b);
        if (vercmp)
                return vercmp;
-       arch_prio1 = pkg_get_int(a, PKG_ARCH_PRIORITY);
-       arch_prio2 = pkg_get_int(b, PKG_ARCH_PRIORITY);
+       arch_prio1 = pkg_get_arch_priority(a);
+       arch_prio2 = pkg_get_arch_priority(b);
        if (!arch_prio1 || !arch_prio2) {
                opkg_msg(ERROR,
                         "Internal error: a->arch_priority=%i b->arch_priority=%i.\n",
@@ -1301,7 +1354,7 @@ int pkg_run_script(pkg_t * pkg, const char *script, const char *args)
 int pkg_arch_supported(pkg_t * pkg)
 {
        nv_pair_list_elt_t *l;
-       char *architecture = pkg_get_string(pkg, PKG_ARCHITECTURE);
+       char *architecture = pkg_get_architecture(pkg);
 
        if (!architecture)
                return 1;

diff --git a/libopkg/pkg.h b/libopkg/pkg.h
index 2c645aade2e5da7e9900453e2083090ed93428cd..df5d7a76086c6709d8dcd63293f3dd30e8123701 100644 (file)
--- a/libopkg/pkg.h
+++ b/libopkg/pkg.h
@@ -89,8 +89,6 @@ enum pkg_fields {
        PKG_LOCAL_FILENAME,
        PKG_VERSION,
        PKG_REVISION,
-       PKG_ARCHITECTURE,
-       PKG_ARCH_PRIORITY,
        PKG_DESCRIPTION,
        PKG_MD5SUM,
        PKG_SHA256SUM,
@@ -168,6 +166,8 @@ struct pkg {
        int auto_installed:1;
        int is_upgrade:1;
 
+       int arch_index:3;
+
        struct blob_buf blob;
 };
 
@@ -206,6 +206,11 @@ static inline void * pkg_get_ptr(const pkg_t *pkg, int id)
        return ptr ? *ptr : NULL;
 }
 
+char *pkg_set_architecture(pkg_t *pkg, const char *architecture, ssize_t len);
+char *pkg_get_architecture(const pkg_t *pkg);
+int pkg_get_arch_priority(const pkg_t *pkg);
+
+
 abstract_pkg_t *abstract_pkg_new(void);
 
 /*

diff --git a/libopkg/pkg_depends.c b/libopkg/pkg_depends.c
index ebd763a8d7e4d4bb48b5f8012b8fa063d99679b6..b2e0df0c0ef45add09519f0ddac3b4f4778a309c 100644 (file)
--- a/libopkg/pkg_depends.c
+++ b/libopkg/pkg_depends.c
@@ -510,10 +510,10 @@ static int is_pkg_in_pkg_vec(pkg_vec_t * vec, pkg_t * pkg)
        int i;
        char *arch1, *arch2;
        pkg_t **pkgs = vec->pkgs;
-       arch1 = pkg_get_string(pkg, PKG_ARCHITECTURE);
+       arch1 = pkg_get_architecture(pkg);
 
        for (i = 0; i < vec->len; i++) {
-               arch2 = pkg_get_string(*(pkgs + i), PKG_ARCHITECTURE);
+               arch2 = pkg_get_architecture(*(pkgs + i));
 
                if ((strcmp(pkg->name, (*(pkgs + i))->name) == 0)
                    && (pkg_compare_versions(pkg, *(pkgs + i)) == 0)

diff --git a/libopkg/pkg_hash.c b/libopkg/pkg_hash.c
index a8809a3025d3cc20f0e0d241f36c28d0c463a7bc..f5c049507848e31748624079e69e7522ceb9abb8 100644 (file)
--- a/libopkg/pkg_hash.c
+++ b/libopkg/pkg_hash.c
@@ -141,8 +141,7 @@ pkg_hash_add_from_file(const char *file_name,
                        continue;
                }
 
-               if (!pkg_get_string(pkg, PKG_ARCHITECTURE) ||
-                   !pkg_get_int(pkg, PKG_ARCH_PRIORITY)) {
+               if (!pkg_get_architecture(pkg) || !pkg_get_arch_priority(pkg)) {
                        char *version_str = pkg_version_str_alloc(pkg);
                        opkg_msg(NOTICE, "Package %s version %s has no "
                                 "valid architecture, ignoring.\n",
@@ -319,11 +318,11 @@ pkg_t *pkg_hash_fetch_best_installation_candidate(abstract_pkg_t * apkg,
                        /* count packages matching max arch priority and keep track of last one */
                        for (j = 0; j < vec->len; j++) {
                                pkg_t *maybe = vec->pkgs[j];
-                               arch_priority = pkg_get_int(maybe, PKG_ARCH_PRIORITY);
+                               arch_priority = pkg_get_arch_priority(maybe);
 
                                opkg_msg(DEBUG,
                                         "%s arch=%s arch_priority=%d version=%s.\n",
-                                        maybe->name, pkg_get_string(maybe, PKG_ARCHITECTURE),
+                                        maybe->name, pkg_get_architecture(maybe),
                                         arch_priority, pkg_get_string(maybe, PKG_VERSION));
                                /* We make sure not to add the same package twice. Need to search for the reason why
                                   they show up twice sometimes. */
@@ -393,7 +392,7 @@ pkg_t *pkg_hash_fetch_best_installation_candidate(abstract_pkg_t * apkg,
                int prio = 0;
                for (i = 0; i < matching_pkgs->len; i++) {
                        pkg_t *matching = matching_pkgs->pkgs[i];
-                       arch_priority = pkg_get_int(matching, PKG_ARCH_PRIORITY);
+                       arch_priority = pkg_get_arch_priority(matching);
                        if (arch_priority > prio) {
                                priorized_matching = matching;
                                prio = arch_priority;
@@ -411,7 +410,7 @@ pkg_t *pkg_hash_fetch_best_installation_candidate(abstract_pkg_t * apkg,
                        pkg_t *matching = matching_pkgs->pkgs[i];
                        opkg_msg(INFO, "%s %s %s\n",
                                 matching->name, pkg_get_string(matching, PKG_VERSION),
-                                pkg_get_string(matching, PKG_ARCHITECTURE));
+                                pkg_get_architecture(matching));
                }
        }
 
@@ -437,7 +436,7 @@ pkg_t *pkg_hash_fetch_best_installation_candidate(abstract_pkg_t * apkg,
        if (priorized_matching) {
                opkg_msg(INFO, "Using priorized matching %s %s %s.\n",
                         priorized_matching->name, pkg_get_string(priorized_matching, PKG_VERSION),
-                        pkg_get_string(priorized_matching, PKG_ARCHITECTURE));
+                        pkg_get_architecture(priorized_matching));
                return priorized_matching;
        }
        if (nmatching > 1) {
@@ -448,7 +447,7 @@ pkg_t *pkg_hash_fetch_best_installation_candidate(abstract_pkg_t * apkg,
        if (latest_matching) {
                opkg_msg(INFO, "Using latest matching %s %s %s.\n",
                         latest_matching->name, pkg_get_string(latest_matching, PKG_VERSION),
-                        pkg_get_string(latest_matching, PKG_ARCHITECTURE));
+                        pkg_get_architecture(latest_matching));
                return latest_matching;
        }
        return NULL;

diff --git a/libopkg/pkg_parse.c b/libopkg/pkg_parse.c
index cfa551e100bf82c98dab243677e5287befea3bea..e0c0069f40ab8a37a1d93743c5cb4908732cadf0 100644 (file)
--- a/libopkg/pkg_parse.c
+++ b/libopkg/pkg_parse.c
@@ -28,6 +28,7 @@
 #include "pkg_depends.h"
 #include "libbb/libbb.h"
 
+#include "file_util.h"
 #include "parse_util.h"
 
 static void parse_status(pkg_t * pkg, const char *sstr)
@@ -94,16 +95,20 @@ int parse_version(pkg_t * pkg, const char *vstr)
        return 0;
 }
 
-static int get_arch_priority(const char *arch)
+static char *parse_architecture(pkg_t *pkg, const char *str)
 {
-       nv_pair_list_elt_t *l;
+       const char *s = str;
+       const char *e;
 
-       list_for_each_entry(l, &conf->arch_list.head, node) {
-               nv_pair_t *nv = (nv_pair_t *) l->data;
-               if (strcmp(nv->name, arch) == 0)
-                       return strtol(nv->value, NULL, 0);
-       }
-       return 0;
+       while (isspace(*s))
+               s++;
+
+       e = s + strlen(s);
+
+       while (e > s && isspace(*e))
+               e--;
+
+       return pkg_set_architecture(pkg, s, e - s);
 }
 
 int pkg_parse_line(void *ptr, const char *line, uint mask)
@@ -114,6 +119,7 @@ int pkg_parse_line(void *ptr, const char *line, uint mask)
        /* these flags are a bit hackish... */
        static int reading_conffiles = 0, reading_description = 0;
        static char *description = NULL;
+       char *s;
        int ret = 0;
 
        /* Exclude globally masked fields. */
@@ -124,11 +130,9 @@ int pkg_parse_line(void *ptr, const char *line, uint mask)
 
        switch (*line) {
        case 'A':
-               if ((mask & PFM_ARCHITECTURE) && is_field("Architecture", line)) {
-                       pkg_set_int(pkg, PKG_ARCH_PRIORITY, get_arch_priority(
-                               pkg_set_string(pkg, PKG_ARCHITECTURE, line + strlen("Architecture") + 1)));
-
-               } else if ((mask & PFM_AUTO_INSTALLED)
+               if ((mask & PFM_ARCHITECTURE) && is_field("Architecture", line))
+                       parse_architecture(pkg, line + strlen("Architecture") + 1);
+               else if ((mask & PFM_AUTO_INSTALLED)
                           && is_field("Auto-Installed", line)) {
                        char *tmp = parse_simple("Auto-Installed", line);
                        if (strcmp(tmp, "yes") == 0)
@@ -182,12 +186,12 @@ int pkg_parse_line(void *ptr, const char *line, uint mask)
                break;
 
        case 'M':
-               if ((mask & PFM_MD5SUM) && is_field("MD5sum:", line))
-                       pkg_set_string(pkg, PKG_MD5SUM, line + strlen("MD5sum") + 1);
-               /* The old opkg wrote out status files with the wrong
-                * case for MD5sum, let's parse it either way */
-               else if ((mask & PFM_MD5SUM) && is_field("MD5Sum:", line))
-                       pkg_set_string(pkg, PKG_MD5SUM, line + strlen("MD5Sum") + 1);
+               if ((mask & PFM_MD5SUM) && (is_field("MD5sum:", line) || is_field("MD5Sum:", line))) {
+                       size_t len;
+                       char *cksum = checksum_hex2bin(line + strlen("MD5sum") + 1, &len);
+                       if (cksum && len == 16)
+                               pkg_set_raw(pkg, PKG_MD5SUM, cksum, len);
+               }
                else if ((mask & PFM_MAINTAINER)
                         && is_field("Maintainer", line))
                        pkg_set_string(pkg, PKG_MAINTAINER, line + strlen("Maintainer") + 1);
@@ -216,8 +220,12 @@ int pkg_parse_line(void *ptr, const char *line, uint mask)
                if ((mask & PFM_SECTION) && is_field("Section", line))
                        pkg_set_string(pkg, PKG_SECTION, line + strlen("Section") + 1);
 #ifdef HAVE_SHA256
-               else if ((mask & PFM_SHA256SUM) && is_field("SHA256sum", line))
-                       pkg_set_string(pkg, PKG_SHA256SUM, line + strlen("SHA256sum") + 1);
+               else if ((mask & PFM_SHA256SUM) && is_field("SHA256sum", line)) {
+                       size_t len;
+                       char *cksum = checksum_hex2bin(line + strlen("SHA256sum") + 1, &len);
+                       if (cksum && len == 32)
+                               pkg_set_raw(pkg, PKG_SHA256SUM, cksum, len);
+               }
 #endif
                else if ((mask & PFM_SIZE) && is_field("Size", line)) {
                        pkg_set_int(pkg, PKG_SIZE, strtoul(line + strlen("Size") + 1, NULL, 0));

diff --git a/libopkg/pkg_vec.c b/libopkg/pkg_vec.c
index d37a18505abedcc3f70755674656bb6b3b82d3d5..a60c5ab7f8bcc0fa60f56b315d1f8d97a0fef7f9 100644 (file)
--- a/libopkg/pkg_vec.c
+++ b/libopkg/pkg_vec.c
@@ -53,12 +53,12 @@ void pkg_vec_insert_merge(pkg_vec_t * vec, pkg_t * pkg, int set_status)
        int i;
        int found = 0;
        char *pkg_version = pkg_get_string(pkg, PKG_VERSION);
-       char *pkg_architecture = pkg_get_string(pkg, PKG_ARCHITECTURE);
+       char *pkg_architecture = pkg_get_architecture(pkg);
        char *vec_architecture;
 
        /* look for a duplicate pkg by name, version, and architecture */
        for (i = 0; i < vec->len; i++) {
-               vec_architecture = pkg_get_string(vec->pkgs[i], PKG_ARCHITECTURE);
+               vec_architecture = pkg_get_architecture(vec->pkgs[i]);
 
                opkg_msg(DEBUG2, "%s %s arch=%s vs. %s %s arch=%s.\n",
                         pkg->name, pkg_version, pkg_architecture,

diff --git a/tests/libopkg_test.c b/tests/libopkg_test.c
index ab15eb79a5d0c4b4f49b2de1e0c39b230ad66c56..0d6a703ea2764703420be21b4a5648c20ace99a8 100644 (file)
--- a/tests/libopkg_test.c
+++ b/tests/libopkg_test.c
@@ -70,7 +70,7 @@ void print_package(pkg_t * pkg)
               pkg->name,
               v,
               pkg->src->name,
-              pkg_get_string(pkg, PKG_ARCHITECTURE),
+              pkg_get_architecture(pkg),
               pkg_get_string(pkg, PKG_DESCRIPTION),
               tags ? tags : "",
               (unsigned long) pkg_get_int(pkg, PKG_SIZE), pkg->state_status);
@@ -93,7 +93,7 @@ void opkg_test(void)
                pkg =
                    opkg_find_package(find_pkg->name,
                                      pkg_get_string(find_pkg, PKG_VERSION),
-                                     pkg_get_string(find_pkg, PKG_ARCHITECTURE),
+                                     pkg_get_architecture(find_pkg),
                                      find_pkg->src->name);
                if (pkg) {
                        print_package(pkg);