Skip to content

Commit

Permalink
Fix hashmap with dna key that was simply cast from the float64 value.
Browse files Browse the repository at this point in the history 
* this might fix a problem with failures in 1.81.0 on 20-23 of hash_reg
  on architectures like armvhl, aarch64, ppc64le, s390x.
* fix bug in hash_reg.c; add test for identical content between
  aset and hashmap operations on dna.
* also add test of writing/reading in jp2kio_reg  with J2K codec.
  • Loading branch information
@DanBloomberg
DanBloomberg committed Jun 9, 2021
1 parent 6dfffd6 commit b215a9a
Show file tree
Hide file tree
Showing 8 changed files with 121 additions and 71 deletions.
97 changes: 45 additions & 52 deletions prog/hash_reg.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,11 @@ const l_int32 da_intersection = 16001;
l_int32 main(int argc,
char **argv)
{
l_uint8 *data1, *data2;
l_int32 i, n, c1, c2, s1;
size_t size1, size2;
l_uint8 *data1;
l_int32 i, n, c1, c2, c3, c4, c5, s1;
size_t size1;
L_ASET *set;
L_DNA *da0, *da1, *da2, *da3;
L_DNA *da0, *da1, *da2, *da3, *da4, *da5;
L_HASHMAP *hmap;
PTA *pta0, *pta1, *pta2, *pta3;
SARRAY *sa0, *sa1, *sa2, *sa3;
Expand Down Expand Up @@ -96,20 +96,17 @@ L_REGPARAMS *rp;
c1 = sarrayGetCount(sa3);
sarrayDestroy(&sa3);
regTestCompareValues(rp, string_set, c1, 0); /* 1 */
if (rp->display)
lept_stderr(" aset: size without dups = %d\n", c1);
if (rp->display) lept_stderr(" aset: size without dups = %d\n", c1);
sarrayIntersectionByAset(sa1, sa2, &sa3);
c1 = sarrayGetCount(sa3);
sarrayDestroy(&sa3);
regTestCompareValues(rp, string_intersection, c1, 0); /* 2 */
if (rp->display)
lept_stderr(" aset: intersection size = %d\n", c1);
if (rp->display) lept_stderr(" aset: intersection size = %d\n", c1);
sarrayUnionByAset(sa1, sa2, &sa3);
c1 = sarrayGetCount(sa3);
sarrayDestroy(&sa3);
regTestCompareValues(rp, string_union, c1, 0); /* 3 */
if (rp->display)
lept_stderr(" aset: union size = %d\n", c1);
if (rp->display) lept_stderr(" aset: union size = %d\n", c1);

/* Test string hashing with hashmap */
hmap = l_hmapCreateFromSarray(sa1);
Expand All @@ -122,20 +119,17 @@ L_REGPARAMS *rp;
c1 = sarrayGetCount(sa3);
sarrayDestroy(&sa3);
regTestCompareValues(rp, string_set, c1, 0); /* 5 */
if (rp->display)
lept_stderr(" hmap: size without dups = %d\n", c1);
if (rp->display) lept_stderr(" hmap: size without dups = %d\n", c1);
sarrayIntersectionByHmap(sa1, sa2, &sa3);
c1 = sarrayGetCount(sa3);
sarrayDestroy(&sa3);
regTestCompareValues(rp, string_intersection, c1, 0); /* 6 */
if (rp->display)
lept_stderr(" hmap: intersection size = %d\n", c1);
if (rp->display) lept_stderr(" hmap: intersection size = %d\n", c1);
sarrayUnionByHmap(sa1, sa2, &sa3);
c1 = sarrayGetCount(sa3);
sarrayDestroy(&sa3);
regTestCompareValues(rp, string_union, c1, 0); /* 7 */
if (rp->display)
lept_stderr(" hmap: union size = %d\n", c1);
if (rp->display) lept_stderr(" hmap: union size = %d\n", c1);
sarrayDestroy(&sa3);
sarrayDestroy(&sa0);
sarrayDestroy(&sa1);
Expand All @@ -162,20 +156,17 @@ L_REGPARAMS *rp;
c1 = ptaGetCount(pta3);
ptaDestroy(&pta3);
regTestCompareValues(rp, pta_set, c1, 0); /* 9 */
if (rp->display)
lept_stderr(" aset: size without dups = %d\n", c1);
if (rp->display) lept_stderr(" aset: size without dups = %d\n", c1);
ptaIntersectionByAset(pta1, pta2, &pta3);
c1 = ptaGetCount(pta3);
ptaDestroy(&pta3);
regTestCompareValues(rp, pta_intersection, c1, 0); /* 10 */
if (rp->display)
lept_stderr(" aset: intersection size = %d\n", c1);
if (rp->display) lept_stderr(" aset: intersection size = %d\n", c1);
ptaUnionByAset(pta1, pta2, &pta3);
c1 = ptaGetCount(pta3);
ptaDestroy(&pta3);
regTestCompareValues(rp, pta_union, c1, 0); /* 11 */
if (rp->display)
lept_stderr(" aset: union size = %d\n", c1);
if (rp->display) lept_stderr(" aset: union size = %d\n", c1);

/* Test point hashing with hashmap */
hmap = l_hmapCreateFromPta(pta2);
Expand All @@ -187,20 +178,17 @@ L_REGPARAMS *rp;
c1 = ptaGetCount(pta3);
ptaDestroy(&pta3);
regTestCompareValues(rp, pta_set, c1, 0); /* 13 */
if (rp->display)
lept_stderr(" hmap: size without dups = %d\n", c1);
if (rp->display) lept_stderr(" hmap: size without dups = %d\n", c1);
ptaIntersectionByHmap(pta1, pta2, &pta3);
c1 = ptaGetCount(pta3);
ptaDestroy(&pta3);
regTestCompareValues(rp, pta_intersection, c1, 0); /* 14 */
if (rp->display)
lept_stderr(" hmap: intersection size = %d\n", c1);
if (rp->display) lept_stderr(" hmap: intersection size = %d\n", c1);
ptaUnionByHmap(pta1, pta2, &pta3);
c1 = ptaGetCount(pta3);
ptaDestroy(&pta3);
regTestCompareValues(rp, pta_union, c1, 0); /* 15 */
if (rp->display)
lept_stderr(" hmap: union size = %d\n", c1);
if (rp->display) lept_stderr(" hmap: union size = %d\n", c1);
ptaDestroy(&pta0);
ptaDestroy(&pta1);
ptaDestroy(&pta2);
Expand Down Expand Up @@ -228,20 +216,17 @@ L_REGPARAMS *rp;
c1 = l_dnaGetCount(da3);
l_dnaDestroy(&da3);
regTestCompareValues(rp, da_set, c1, 0); /* 17 */
if (rp->display)
lept_stderr(" aset: size without dups = %d\n", c1);
if (rp->display) lept_stderr(" aset: size without dups = %d\n", c1);
l_dnaIntersectionByAset(da1, da2, &da3);
c1 = l_dnaGetCount(da3);
l_dnaDestroy(&da3);
regTestCompareValues(rp, da_intersection, c1, 0); /* 18 */
if (rp->display)
lept_stderr(" aset: intersection size = %d\n", c1);
if (rp->display) lept_stderr(" aset: intersection size = %d\n", c1);
l_dnaUnionByAset(da1, da2, &da3);
c1 = l_dnaGetCount(da3);
l_dnaDestroy(&da3);
regTestCompareValues(rp, da_union, c1, 0); /* 19 */
if (rp->display)
lept_stderr(" aset: union size = %d\n", c1);
if (rp->display) lept_stderr(" aset: union size = %d\n", c1);

/* Test dna hashing with hashmap */
hmap = l_hmapCreateFromDna(da2);
Expand All @@ -253,26 +238,23 @@ L_REGPARAMS *rp;
c1 = l_dnaGetCount(da3);
l_dnaDestroy(&da3);
regTestCompareValues(rp, da_set, c1, 0); /* 21 */
if (rp->display)
lept_stderr(" hmap: size without dups = %d\n", c1);
if (rp->display) lept_stderr(" hmap: size without dups = %d\n", c1);
l_dnaIntersectionByHmap(da1, da2, &da3);
c1 = l_dnaGetCount(da3);
l_dnaDestroy(&da3);
regTestCompareValues(rp, da_intersection, c1, 0); /* 22 */
if (rp->display)
lept_stderr(" hmap: intersection size = %d\n", c1);
if (rp->display) lept_stderr(" hmap: intersection size = %d\n", c1);
l_dnaUnionByHmap(da1, da2, &da3);
c1 = l_dnaGetCount(da3);
l_dnaDestroy(&da3);
regTestCompareValues(rp, da_union, c1, 0); /* 23 */
if (rp->display)
lept_stderr(" hmap: union size = %d\n", c1);
if (rp->display) lept_stderr(" hmap: union size = %d\n", c1);
l_dnaDestroy(&da0);
l_dnaDestroy(&da1);
l_dnaDestroy(&da2);

/* Test dna hashing, comparing ordered and unordered, and
* comparing the results. */
/* Another test of dna hashing, showing equivalence
* of results between ordered and unordered sets. */
da0 = l_dnaMakeSequence(0, 1, 20);
n = l_dnaGetCount(da0);
for (i = 0; i < n; i++)
Expand All @@ -284,25 +266,36 @@ L_REGPARAMS *rp;
c1 = l_dnaGetCount(da1);
c2 = l_dnaGetCount(da2);
l_dnaRemoveDupsByAset(da2, &da3);
l_dnaWriteMem(&data1, &size1, da3);
l_dnaDestroy(&da3);
l_dnaRemoveDupsByHmap(da2, &da3, NULL);
l_dnaWriteMem(&data2, &size2, da3);
regTestCompareStrings(rp, data1, size1, data2, size2); /* 24 */
if (rp->display) lept_stderr("%s", data1);
lept_free(data1);
lept_free(data2);
l_dnaRemoveDupsByHmap(da2, &da4, NULL);
/* Show the two sets da3 and da4 are identical in content,
* because they have the same size and their intersection also
* has the same size. */
c3 = l_dnaGetCount(da3);
c4 = l_dnaGetCount(da4);
regTestCompareValues(rp, c3, c4, 0); /* 24 */
l_dnaIntersectionByHmap(da3, da4, &da5);
c5 = l_dnaGetCount(da5);
regTestCompareValues(rp, c4, c5, 0); /* 25 */
if (rp->display) {
lept_stderr("\nc1 = %d, c2 = %d\n", c1, c2);
lept_stderr("c3 = %d, c4 = %d, c5 = %d\n", c3, c4, c5);
l_dnaWriteMem(&data1, &size1, da4);
lept_stderr("%s", data1);
lept_free(data1);
}
l_dnaDestroy(&da0);
l_dnaDestroy(&da1);
l_dnaDestroy(&da2);
l_dnaDestroy(&da3);
l_dnaDestroy(&da4);
l_dnaDestroy(&da5);

/* Test pixel counting operations with hashmap and ordered map */
pix1 = pixRead("wet-day.jpg");
pixCountRGBColorsByHash(pix1, &c1);
pixCountRGBColors(pix1, 1, &c2);
regTestCompareValues(rp, 42427, c1, 0); /* 24 */
regTestCompareValues(rp, 42427, c2, 0); /* 25 */
regTestCompareValues(rp, 42427, c1, 0); /* 26 */
regTestCompareValues(rp, 42427, c2, 0); /* 27 */
if (rp->display) {
lept_stderr("Color count using hashmap: %d\n", c1);
lept_stderr("Color count using aset: %d\n", c2);
Expand Down
43 changes: 32 additions & 11 deletions prog/jp2kio_reg.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@

void DoJp2kTest1(L_REGPARAMS *rp, const char *fname);
void DoJp2kTest2(L_REGPARAMS *rp, const char *fname);
void DoJp2kTest3(L_REGPARAMS *rp, const char *fname);


int main(int argc,
Expand Down Expand Up @@ -84,6 +85,7 @@ L_REGPARAMS *rp;
DoJp2kTest1(rp, "test24.jpg");
/* DoJp2kTest2(rp, "karen8.jpg"); */ /* encode fails on smallest image */
DoJp2kTest2(rp, "test24.jpg");
DoJp2kTest3(rp, "wyom.jpg");
return regTestCleanup(rp);
}

Expand All @@ -101,10 +103,10 @@ PIX *pix0, *pix1, *pix2, *pix3;
pix0 = pixRead(fname);
pix1 = pixScale(pix0, 0.5, 0.5);
pixGetDimensions(pix1, &w, &h, NULL);
regTestWritePixAndCheck(rp, pix1, IFF_JP2);
regTestWritePixAndCheck(rp, pix1, IFF_JP2); /* 0, 5 */
name = regTestGenLocalFilename(rp, -1, IFF_JP2);
pix2 = pixRead(name);
regTestWritePixAndCheck(rp, pix2, IFF_JP2);
regTestWritePixAndCheck(rp, pix2, IFF_JP2); /* 1, 6 */
pixDisplayWithTitle(pix2, 0, 100, "1", rp->display);
pixDestroy(&pix1);
pixDestroy(&pix2);
Expand All @@ -115,12 +117,12 @@ PIX *pix0, *pix1, *pix2, *pix3;
snprintf(buf, sizeof(buf), "/tmp/lept/regout/jp2kio.%02d.jp2",
rp->index + 1);
pixWriteJp2k(buf, pix1, 38, 0, 0, 0); /* write cropped to the box */
regTestCheckFile(rp, buf);
regTestCheckFile(rp, buf); /* 2, 7 */
pix2 = pixRead(buf); /* read the cropped image */
regTestWritePixAndCheck(rp, pix2, IFF_JP2);
regTestWritePixAndCheck(rp, pix2, IFF_JP2); /* 3, 8 */
pixDisplayWithTitle(pix2, 500, 100, "2", rp->display);
pix3 = pixReadJp2k(buf, 2, NULL, 0, 0); /* read cropped image at 2x red */
regTestWritePixAndCheck(rp, pix3, IFF_JP2);
regTestWritePixAndCheck(rp, pix3, IFF_JP2); /* 4, 9 */
pixDisplayWithTitle(pix3, 1000, 100, "3", rp->display);
pixDestroy(&pix0);
pixDestroy(&pix1);
Expand All @@ -146,13 +148,13 @@ PIX *pix0, *pix1, *pix2, *pix3;
pix0 = pixRead(fname);
pix1 = pixScale(pix0, 0.5, 0.5);
pixGetDimensions(pix1, &w, &h, NULL);
regTestWritePixAndCheck(rp, pix1, IFF_JP2);
regTestWritePixAndCheck(rp, pix1, IFF_JP2); /* 10 */
name = regTestGenLocalFilename(rp, -1, IFF_JP2);
pix2 = pixRead(name);
regTestWritePixAndCheck(rp, pix2, IFF_JP2);
regTestWritePixAndCheck(rp, pix2, IFF_JP2); /* 11 */
data = l_binaryRead(name, &nbytes);
pix3 = pixReadMemJp2k(data, nbytes, 1, NULL, 0, 0);
regTestWritePixAndCheck(rp, pix3, IFF_JP2);
regTestWritePixAndCheck(rp, pix3, IFF_JP2); /* 12 */
pixDisplayWithTitle(pix3, 0, 100, "1", rp->display);
pixDestroy(&pix1);
pixDestroy(&pix2);
Expand All @@ -165,13 +167,13 @@ PIX *pix0, *pix1, *pix2, *pix3;
snprintf(buf, sizeof(buf), "/tmp/lept/regout/jp2kio.%02d.jp2",
rp->index + 1);
pixWriteJp2k(buf, pix1, 38, 0, 0, 0); /* write cropped to the box */
regTestCheckFile(rp, buf);
regTestCheckFile(rp, buf); /* 13 */
data = l_binaryRead(buf, &nbytes);
pix2 = pixReadMemJp2k(data, nbytes, 1, NULL, 0, 0); /* read it again */
regTestWritePixAndCheck(rp, pix2, IFF_JP2);
regTestWritePixAndCheck(rp, pix2, IFF_JP2); /* 14 */
pixDisplayWithTitle(pix2, 500, 100, "2", rp->display);
pix3 = pixReadMemJp2k(data, nbytes, 2, NULL, 0, 0); /* read at 2x red */
regTestWritePixAndCheck(rp, pix3, IFF_JP2);
regTestWritePixAndCheck(rp, pix3, IFF_JP2); /* 15 */
pixDisplayWithTitle(pix3, 1000, 100, "3", rp->display);
boxDestroy(&box);
pixDestroy(&pix0);
Expand All @@ -182,3 +184,22 @@ PIX *pix0, *pix1, *pix2, *pix3;
lept_free(name);
return;
}

void DoJp2kTest3(L_REGPARAMS *rp,
const char *fname)
{
FILE *fp;
PIX *pix0, *pix1;

/* Test write and read using J2K codec */
lept_mkdir("lept/jp2k");
pix0 = pixRead(fname);
fp = fopenWriteStream("/tmp/lept/jp2k/wyom.j2k", "wb+");
pixWriteStreamJp2k(fp, pix0, 34, 4, L_J2K_CODEC, 0, 0);
fclose(fp);
pix1 = pixRead("/tmp/lept/jp2k/wyom.j2k");
regTestCompareSimilarPix(rp, pix0, pix1, 20, 0.01, 0); /* 16 */
pixDisplayWithTitle(pix1, 500, 500, NULL, rp->display);
pixDestroy(&pix0);
pixDestroy(&pix1);
}
1 change: 1 addition & 0 deletions src/allheaders.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2658,6 +2658,7 @@ LEPT_DLL extern l_int32 lept_roundftoi ( l_float32 fval );
LEPT_DLL extern l_ok l_hashStringToUint64 ( const char *str, l_uint64 *phash );
LEPT_DLL extern l_ok l_hashStringToUint64Fast ( const char *str, l_uint64 *phash );
LEPT_DLL extern l_ok l_hashPtToUint64 ( l_int32 x, l_int32 y, l_uint64 *phash );
LEPT_DLL extern l_ok l_hashFloat64ToUint64 ( l_float64 val, l_uint64 *phash );
LEPT_DLL extern l_ok findNextLargerPrime ( l_int32 start, l_uint32 *pprime );
LEPT_DLL extern l_ok lept_isPrime ( l_uint64 n, l_int32 *pis_prime, l_uint32 *pfactor );
LEPT_DLL extern l_uint32 convertIntToGrayCode ( l_uint32 val );
Expand Down
8 changes: 5 additions & 3 deletions src/dnafunc1.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -517,7 +517,7 @@ L_DNA *da_small, *da_big, *dad;
*
* <pre>
* Notes:
* (1) Use the values in %da as the hash keys.
* (1) Derive the hash keys from the values in %da.
* (2) The indices into %da are stored in the val field of the hashitems.
* This is necessary so that %hmap and %da can be used together.
* </pre>
Expand All @@ -540,7 +540,8 @@ L_HASHMAP *hmap;
hmap = l_hmapCreate(0, 0);
for (i = 0; i < n; i++) {
l_dnaGetDValue(da, i, &dval);
hitem = l_hmapLookup(hmap, (l_uint64)dval, i, L_HMAP_CREATE);
l_hashFloat64ToUint64(dval, &key);
hitem = l_hmapLookup(hmap, key, i, L_HMAP_CREATE);
}
return hmap;
}
Expand Down Expand Up @@ -699,7 +700,8 @@ L_HASHMAP *hmap;
n = l_dnaGetCount(da_small);
for (i = 0; i < n; i++) {
l_dnaGetDValue(da_small, i, &dval);
hitem = l_hmapLookup(hmap, (l_uint64)dval, i, L_HMAP_CHECK);
l_hashFloat64ToUint64(dval, &key);
hitem = l_hmapLookup(hmap, key, i, L_HMAP_CHECK);
if (!hitem || hitem->count == 0)
continue;
l_dnaAddNumber(dad, dval);
Expand Down
10 changes: 5 additions & 5 deletions src/jp2kheader.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ static const l_int32 MAX_JP2K_HEIGHT = 100000;
* \param[out] ph [optional]
* \param[out] pbps [optional] bits/sample
* \param[out] pspp [optional] samples/pixel
* \param[out] pcodec [optional] JP2_CODEC or J2K_CODEC
* \param[out] pcodec [optional] L_JP2_CODEC or L_J2K_CODEC
* \return 0 if OK, 1 on error
*/
l_ok
Expand Down Expand Up @@ -108,7 +108,7 @@ FILE *fp;
* \param[out] ph [optional]
* \param[out] pbps [optional] bits/sample
* \param[out] pspp [optional] samples/pixel
* \param[out] pcodec [optional] JP2_CODEC or J2K_CODEC
* \param[out] pcodec [optional] L_JP2_CODEC or L_J2K_CODEC
* \return 0 if OK, 1 on error
*/
l_ok
Expand Down Expand Up @@ -147,23 +147,23 @@ l_int32 nread, ret;
* \param[out] ph [optional]
* \param[out] pbps [optional] bits/sample
* \param[out] pspp [optional] samples/pixel
* \param[out] pcodec [optional] JP2_CODEC or J2K_CODEC
* \param[out] pcodec [optional] L_JP2_CODEC or L_J2K_CODEC
* \return 0 if OK, 1 on error
*
* <pre>
* Notes:
* (1) The ISO/IEC reference for jpeg2000 is
* http://www.jpeg.org/public/15444-1annexi.pdf
* and the file format syntax begins at page 127.
* (2) With a image file codec (JP2_CODEC), the Image Header Box
* (2) With a image file codec (L_JP2_CODEC), the Image Header Box
* begins with 'ihdr' = 0x69686472 in big-endian order. This
* typically, but not always, starts on byte 44, with the
* big-endian data fields beginning at byte 48:
* h: 4 bytes
* w: 4 bytes
* spp: 2 bytes
* bps: 1 byte (contains bps - 1)
* (3) With a codestream codec (J2K_CODEC), the first 4 bytes are
* (3) With a codestream codec (L_J2K_CODEC), the first 4 bytes are
* 0xff4fff51. The fields for w and h appear to start on byte 8,
* and the fields for spp and bps appear to start on byte 40.
* </pre>
Expand Down
Loading

0 comments on commit b215a9a

Please sign in to comment.