[prev in list] [next in list] [prev in thread] [next in thread]
List: monetdb-checkins
Subject: MonetDB: partition - Add hash-partitioning producing candidate l...
From: Martin Kersten <commits+mk=cwi.nl () monetdb ! org>
Date: 2017-12-30 10:20:08
Message-ID: hg.01c31162353f.1514629208.6315528441665844383 () monetdb-vm0 ! spin-off ! cwi ! nl
[Download RAW message or body]
Changeset: 01c31162353f for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=01c31162353f
Modified Files:
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/exports.stable.out
monetdb5/modules/mal/Tests/hashpartition.malC
monetdb5/modules/mal/Tests/hashpartition.stable.out
monetdb5/modules/mal/partition.c
monetdb5/modules/mal/partition.h
monetdb5/modules/mal/partition.mal
Branch: partition
Log Message:
Add hash-partitioning producing candidate lists
diffs (270 lines):
diff --git a/clients/Tests/MAL-signatures.stable.out \
b/clients/Tests/MAL-signatures.stable.out
--- a/clients/Tests/MAL-signatures.stable.out
+++ b/clients/Tests/MAL-signatures.stable.out
@@ -8228,6 +8228,7 @@ Ready.
[ "optimizer", "wlc", "pattern optimizer.wlc():str ", "OPTwrapper;", "" ]
[ "optimizer", "wlc", "pattern optimizer.wlc(mod:str, fcn:str):str \
", "OPTwrapper;", "Inject the workload capture-replay primitives." ] [ \
"partition", "hash", "pattern partition.hash(b:bat[:any_2]):bat[:any_2]... \
", "PARThash;", "Perform a value based partition " ] +[ \
"partition", "slice", "pattern partition.slice(b:bat[:any_2]):bat[:oid]... \
", "PARTslice;", "Perform a value based partition to produce candidate lists " ] [ \
"pcre", "imatch", "command pcre.imatch(s:str, pat:str):bit \
", "PCREimatch;", "Caseless Perl Compatible Regular Expression pattern matching \
against a string" ] [ "pcre", "index", "command pcre.index(pat:pcre, s:str):int \
", "PCREindex;", "match a pattern, return matched position (or 0 when not found)" ] \
[ "pcre", "match", "command pcre.match(s:str, pat:str):bit ", "PCREmatch;", "Perl \
Compatible Regular Expression pattern matching against a string" ]
diff --git a/clients/Tests/MAL-signatures.stable.out.int128 \
b/clients/Tests/MAL-signatures.stable.out.int128
--- a/clients/Tests/MAL-signatures.stable.out.int128
+++ b/clients/Tests/MAL-signatures.stable.out.int128
@@ -10590,6 +10590,7 @@ Ready.
[ "optimizer", "wlc", "pattern optimizer.wlc():str ", "OPTwrapper;", "" ]
[ "optimizer", "wlc", "pattern optimizer.wlc(mod:str, fcn:str):str \
", "OPTwrapper;", "Inject the workload capture-replay primitives." ] [ \
"partition", "hash", "pattern partition.hash(b:bat[:any_2]):bat[:any_2]... \
", "PARThash;", "Perform a value based partition " ] +[ \
"partition", "slice", "pattern partition.slice(b:bat[:any_2]):bat[:oid]... \
", "PARTslice;", "Perform a value based partition to produce candidate lists " ] [ \
"pcre", "imatch", "command pcre.imatch(s:str, pat:str):bit \
", "PCREimatch;", "Caseless Perl Compatible Regular Expression pattern matching \
against a string" ] [ "pcre", "index", "command pcre.index(pat:pcre, s:str):int \
", "PCREindex;", "match a pattern, return matched position (or 0 when not found)" ] \
[ "pcre", "match", "command pcre.match(s:str, pat:str):bit ", "PCREmatch;", "Perl \
Compatible Regular Expression pattern matching against a string" ]
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -1597,6 +1597,7 @@ str OPTvolcanoImplementation(Client cntx
str OPTwlcImplementation(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci);
str OPTwrapper(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
str PARThash(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
+str PARTslice(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
str PCREilike2(bit *ret, const str *s, const str *pat);
str PCREilike3(bit *ret, const str *s, const str *pat, const str *esc);
str PCREimatch(bit *ret, const str *val, const str *pat);
diff --git a/monetdb5/modules/mal/Tests/hashpartition.malC \
b/monetdb5/modules/mal/Tests/hashpartition.malC
--- a/monetdb5/modules/mal/Tests/hashpartition.malC
+++ b/monetdb5/modules/mal/Tests/hashpartition.malC
@@ -42,5 +42,18 @@ io.print(c2);
io.print(c3);
io.print(c4);
io.print(c5);
+
+(o1,o2):= partition.slice(b);
+io.print(o1);
+io.print(o2);
+
+(o1,o2,o3,o4,o5):= partition.slice(b);
+io.print(o1);
+io.print(o2);
+io.print(o3);
+io.print(o4);
+io.print(o5);
+
end tst;
+
tst();
diff --git a/monetdb5/modules/mal/Tests/hashpartition.stable.out \
b/monetdb5/modules/mal/Tests/hashpartition.stable.out
--- a/monetdb5/modules/mal/Tests/hashpartition.stable.out
+++ b/monetdb5/modules/mal/Tests/hashpartition.stable.out
@@ -148,6 +148,94 @@ Ready.
[ 3@0, 19 ]
[ 4@0, 24 ]
[ 5@0, 29 ]
+#--------------------------#
+# h t # name
+# void oid # type
+#--------------------------#
+[ 0@0, 0@0 ]
+[ 1@0, 2@0 ]
+[ 2@0, 4@0 ]
+[ 3@0, 6@0 ]
+[ 4@0, 8@0 ]
+[ 5@0, 10@0 ]
+[ 6@0, 12@0 ]
+[ 7@0, 14@0 ]
+[ 8@0, 16@0 ]
+[ 9@0, 18@0 ]
+[ 10@0, 20@0 ]
+[ 11@0, 22@0 ]
+[ 12@0, 24@0 ]
+[ 13@0, 26@0 ]
+[ 14@0, 28@0 ]
+#--------------------------#
+# h t # name
+# void oid # type
+#--------------------------#
+[ 0@0, 1@0 ]
+[ 1@0, 3@0 ]
+[ 2@0, 5@0 ]
+[ 3@0, 7@0 ]
+[ 4@0, 9@0 ]
+[ 5@0, 11@0 ]
+[ 6@0, 13@0 ]
+[ 7@0, 15@0 ]
+[ 8@0, 17@0 ]
+[ 9@0, 19@0 ]
+[ 10@0, 21@0 ]
+[ 11@0, 23@0 ]
+[ 12@0, 25@0 ]
+[ 13@0, 27@0 ]
+[ 14@0, 29@0 ]
+#--------------------------#
+# h t # name
+# void oid # type
+#--------------------------#
+[ 0@0, 0@0 ]
+[ 1@0, 5@0 ]
+[ 2@0, 10@0 ]
+[ 3@0, 15@0 ]
+[ 4@0, 20@0 ]
+[ 5@0, 25@0 ]
+#--------------------------#
+# h t # name
+# void oid # type
+#--------------------------#
+[ 0@0, 1@0 ]
+[ 1@0, 6@0 ]
+[ 2@0, 11@0 ]
+[ 3@0, 16@0 ]
+[ 4@0, 21@0 ]
+[ 5@0, 26@0 ]
+#--------------------------#
+# h t # name
+# void oid # type
+#--------------------------#
+[ 0@0, 2@0 ]
+[ 1@0, 7@0 ]
+[ 2@0, 12@0 ]
+[ 3@0, 17@0 ]
+[ 4@0, 22@0 ]
+[ 5@0, 27@0 ]
+#--------------------------#
+# h t # name
+# void oid # type
+#--------------------------#
+[ 0@0, 3@0 ]
+[ 1@0, 8@0 ]
+[ 2@0, 13@0 ]
+[ 3@0, 18@0 ]
+[ 4@0, 23@0 ]
+[ 5@0, 28@0 ]
+#--------------------------#
+# h t # name
+# void oid # type
+#--------------------------#
+[ 0@0, 4@0 ]
+[ 1@0, 9@0 ]
+[ 2@0, 14@0 ]
+[ 3@0, 19@0 ]
+[ 4@0, 24@0 ]
+[ 5@0, 29@0 ]
# 14:09:25 >
# 14:09:25 > "Done."
diff --git a/monetdb5/modules/mal/partition.c b/monetdb5/modules/mal/partition.c
--- a/monetdb5/modules/mal/partition.c
+++ b/monetdb5/modules/mal/partition.c
@@ -15,6 +15,7 @@
#include "partition.h"
#define MAXPART 256
+
#define hashpartition(TYPE) \
do { p= BATcount(b); \
bi = bat_iterator(b); \
@@ -28,6 +29,19 @@
} \
} while (0)
+#define slicepartition(TYPE) \
+ do { p= BATcount(b); \
+ bi = bat_iterator(b); \
+ for (r=0; r < p; r++) { \
+ TYPE *v = (TYPE *) BUNtloc(bi, r); \
+ TYPE c = mix_##TYPE((*v)); \
+ if( BUNappend(bn[(int) (c % pieces)], (void*) &r, FALSE) != GDK_SUCCEED){ \
+ msg = createException(MAL,"partition.slice","Storage failed");\
+ break;\
+ }\
+ } \
+ } while (0)
+
str
PARThash(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
@@ -97,3 +111,64 @@ PARThash(Client cntxt, MalBlkPtr mb, Mal
BBPunfix(b->batCacheid);
return msg;
}
+
+str
+PARTslice(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+{
+ bat *ret = getArgReference_bat(stk,pci,0);
+ int i, pieces;
+ BAT *b, *bn[MAXPART];
+ BUN r, p;
+ BATiter bi;
+ str msg = MAL_SUCCEED;
+
+
+ (void) cntxt;
+ (void) mb;
+ pieces = pci->retc;
+ if ( pieces >= MAXPART)
+ throw(MAL,"partition.slice","too many partitions");
+ b = BATdescriptor( stk->stk[getArg(pci, pci->retc)].val.ival);
+ if ( b == NULL)
+ throw(MAL, "partition.slice", SQLSTATE(HY002) RUNTIME_OBJECT_MISSING);
+
+ for( i = 0; i < pieces; i++){
+ size_t cap = BATcount(b) / pieces * 1.1;
+ bn[i] = COLnew(0, TYPE_oid, cap, TRANSIENT);
+ if (bn[i] == NULL){
+ for(i--; i>=0; i--)
+ BBPunfix(bn[i]->batCacheid);
+ BBPunfix(b->batCacheid);
+ throw(MAL, "partition.slice", SQLSTATE(HY001) MAL_MALLOC_FAIL);
+ }
+ }
+ /* distribute the elements over multiple partitions */
+ switch(ATOMstorage(b->ttype)){
+ case TYPE_bte:
+ slicepartition(bte);
+ break;
+ case TYPE_sht:
+ slicepartition(sht);
+ break;
+ case TYPE_int:
+ slicepartition(int);
+ break;
+ case TYPE_lng:
+ slicepartition(lng);
+ break;
+#ifdef HAVE_HGE
+ case TYPE_hge:
+ slicepartition(hge);
+ break;
+#endif
+ default:
+ msg = createException(MAL,"partition.slice","Non-supported type");
+ }
+
+ for(i=0; i< pieces; i++){
+ ret = getArgReference_bat(stk,pci,i);
+ BBPkeepref(*ret = bn[i]->batCacheid);
+ }
+ BBPunfix(b->batCacheid);
+ return msg;
+}
diff --git a/monetdb5/modules/mal/partition.h b/monetdb5/modules/mal/partition.h
--- a/monetdb5/modules/mal/partition.h
+++ b/monetdb5/modules/mal/partition.h
@@ -14,5 +14,6 @@
#include "mal_interpreter.h"
mal_export str PARThash(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
+mal_export str PARTslice(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr p);
#endif /* _PART_H */
diff --git a/monetdb5/modules/mal/partition.mal b/monetdb5/modules/mal/partition.mal
--- a/monetdb5/modules/mal/partition.mal
+++ b/monetdb5/modules/mal/partition.mal
@@ -9,3 +9,7 @@ module partition;
pattern partition.hash(b:bat[:any_2]):bat[:any_2]...
address PARThash
comment "Perform a value based partition ";
+
+pattern partition.slice(b:bat[:any_2]):bat[:oid]...
+address PARTslice
+comment "Perform a value based partition to produce candidate lists ";
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic