[prev in list] [next in list] [prev in thread] [next in thread]
List: monetdb-checkins
Subject: MonetDB: scatter - the scattered insert
From: Niels_Nes <commits+niels=cwi.nl () monetdb ! org>
Date: 2021-05-31 10:51:31
Message-ID: hg.40498c26831c.1622458291.-6979084861817554466 () monetdb-vm0 ! spin-off ! cwi ! nl
[Download RAW message or body]
Changeset: 40498c26831c for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/40498c26831c
Modified Files:
clients/Tests/exports.stable.out
gdk/gdk.h
gdk/gdk_batop.c
sql/storage/bat/bat_storage.c
Branch: scatter
Log Message:
the scattered insert
diffs (truncated from 366 to 300 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -203,6 +203,7 @@ void BATtseqbase(BAT *b, oid o);
void BATundo(BAT *b);
BAT *BATunique(BAT *b, BAT *s);
BAT *BATunmask(BAT *b);
+gdk_return BATupdate(BAT *b, BAT *p, BAT *n, bool force) \
__attribute__((__warn_unused_result__)); BBPrec *BBP[N_BBPINIT];
gdk_return BBPaddfarm(const char *dirname, uint32_t rolemask, bool logerror);
void BBPclear(bat bid);
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -972,6 +972,8 @@ gdk_export gdk_return BATdel(BAT *b, BAT
gdk_export gdk_return BATreplace(BAT *b, BAT *p, BAT *n, bool force)
__attribute__((__warn_unused_result__));
+gdk_export gdk_return BATupdate(BAT *b, BAT *p, BAT *n, bool force)
+ __attribute__((__warn_unused_result__));
/* Functions to perform a binary search on a sorted BAT.
* See gdk_search.c for details. */
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -1090,8 +1090,8 @@ BATdel(BAT *b, BAT *d)
* The last in this series is a BATreplace, which replaces all the
* buns mentioned.
*/
-gdk_return
-BATreplace(BAT *b, BAT *p, BAT *n, bool force)
+static gdk_return
+BATappend_or_update(BAT *b, BAT *p, BAT *n, bool mayappend, bool force)
{
lng t0 = GDKusec();
@@ -1116,12 +1116,14 @@ BATreplace(BAT *b, BAT *p, BAT *n, bool
BATiter bi = bat_iterator(b);
BATiter ni = bat_iterator(n);
+#if 0 /* questionable: what if p point outside b, even if !mayappend? */
if (BATcount(b) == 0 ||
(b->tsorted && b->trevsorted &&
n->tsorted && n->trevsorted &&
ATOMcmp(b->ttype, BUNtail(bi, 0), BUNtail(ni, 0)) == 0)) {
return GDK_SUCCEED;
}
+#endif
OIDXdestroy(b);
IMPSdestroy(b);
@@ -1146,7 +1148,8 @@ BATreplace(BAT *b, BAT *p, BAT *n, bool
for (BUN i = 0, j = BATcount(p); i < j; i++) {
oid updid = BUNtoid(p, i);
- if (updid < b->hseqbase || updid >= hseqend) {
+ if (updid < b->hseqbase ||
+ (!mayappend && updid >= hseqend)) {
GDKerror("id out of range\n");
return GDK_FAIL;
}
@@ -1156,8 +1159,20 @@ BATreplace(BAT *b, BAT *p, BAT *n, bool
return GDK_FAIL;
}
+ const void *new = BUNtvar(ni, i);
+
+ if (updid >= BATcount(b)) {
+ assert(mayappend);
+ while (BATcount(b) < updid) {
+ if (BUNappend(b, ATOMnilptr(b->ttype), force) != GDK_SUCCEED)
+ return GDK_FAIL;
+ }
+ if (BUNappend(b, new, force) != GDK_SUCCEED)
+ return GDK_FAIL;
+ continue;
+ }
+
const void *old = BUNtvar(bi, updid);
- const void *new = BUNtvar(ni, i);
bool isnil = atomcmp(new, nil) == 0;
anynil |= isnil;
if (b->tnil &&
@@ -1259,7 +1274,8 @@ BATreplace(BAT *b, BAT *p, BAT *n, bool
for (BUN i = 0, j = BATcount(p); i < j; i++) {
oid updid = BUNtoid(p, i);
- if (updid < b->hseqbase || updid >= hseqend) {
+ if (updid < b->hseqbase ||
+ (!mayappend && updid >= hseqend)) {
GDKerror("id out of range\n");
return GDK_FAIL;
}
@@ -1268,13 +1284,23 @@ BATreplace(BAT *b, BAT *p, BAT *n, bool
GDKerror("updating committed value\n");
return GDK_FAIL;
}
-
+ if (updid >= BATcount(b)) {
+ assert(mayappend);
+ while (BATcount(b) < updid) {
+ if (BUNappend(b, &(msk){false}, force) != GDK_SUCCEED)
+ return GDK_FAIL;
+ }
+ if (BUNappend(b, &(msk){mskGetVal(n, i)}, force) != GDK_SUCCEED)
+ return GDK_FAIL;
+ continue;
+ }
mskSetVal(b, updid, mskGetVal(n, i));
}
} else if (BATtdense(p)) {
oid updid = BUNtoid(p, 0);
- if (updid < b->hseqbase || updid + BATcount(p) > hseqend) {
+ if (updid < b->hseqbase ||
+ (!mayappend && updid + BATcount(p) > hseqend)) {
GDKerror("id out of range\n");
return GDK_FAIL;
}
@@ -1284,6 +1310,19 @@ BATreplace(BAT *b, BAT *p, BAT *n, bool
return GDK_FAIL;
}
+ if (updid >= BATcount(b)) {
+ assert(mayappend);
+ while (BATcount(b) < updid) {
+ if (BUNappend(b, ATOMnilptr(b->ttype), force) != GDK_SUCCEED)
+ return GDK_FAIL;
+ }
+ return BATappend(b, n, NULL, force);
+ }
+ while (updid + BATcount(n) > BATcount(b)) {
+ if (BUNappend(b, ATOMnilptr(b->ttype), force) != GDK_SUCCEED)
+ return GDK_FAIL;
+ }
+
/* we copy all of n, so if there are nils in n we get
* nils in b (and else we don't know) */
b->tnil = n->tnil;
@@ -1402,7 +1441,8 @@ BATreplace(BAT *b, BAT *p, BAT *n, bool
for (BUN i = 0, j = BATcount(p); i < j; i++) {
oid updid = BUNtoid(p, i);
- if (updid < b->hseqbase || updid >= hseqend) {
+ if (updid < b->hseqbase ||
+ (!mayappend && updid >= hseqend)) {
GDKerror("id out of range\n");
return GDK_FAIL;
}
@@ -1412,8 +1452,20 @@ BATreplace(BAT *b, BAT *p, BAT *n, bool
return GDK_FAIL;
}
+ const void *new = BUNtail(ni, i);
+
+ if (updid >= BATcount(b)) {
+ assert(mayappend);
+ while (BATcount(b) < updid) {
+ if (BUNappend(b, ATOMnilptr(b->ttype), force) != GDK_SUCCEED)
+ return GDK_FAIL;
+ }
+ if (BUNappend(b, new, force) != GDK_SUCCEED)
+ return GDK_FAIL;
+ continue;
+ }
+
const void *old = BUNtloc(bi, updid);
- const void *new = BUNtail(ni, i);
bool isnil = atomcmp(new, nil) == 0;
anynil |= isnil;
if (b->tnil &&
@@ -1501,6 +1553,19 @@ BATreplace(BAT *b, BAT *p, BAT *n, bool
return GDK_SUCCEED;
}
+/* replace values from b at locations specified in p with values in n */
+gdk_return
+BATreplace(BAT *b, BAT *p, BAT *n, bool force)
+{
+ return BATappend_or_update(b, p, n, false, force);
+}
+
+/* like BATreplace, but p may specify locations beyond the end of b */
+gdk_return
+BATupdate(BAT *b, BAT *p, BAT *n, bool force)
+{
+ return BATappend_or_update(b, p, n, true, force);
+}
/*
* BAT Selections
diff --git a/sql/storage/bat/bat_storage.c b/sql/storage/bat/bat_storage.c
--- a/sql/storage/bat/bat_storage.c
+++ b/sql/storage/bat/bat_storage.c
@@ -1132,61 +1132,14 @@ delta_append_bat( sql_delta *bat, BAT *o
if (i && (i->ttype == TYPE_msk || mask_cand(i))) {
oi = BATunmask(i);
}
-
- size_t offset = 0;
- if (BATtdense(offsets)) {
- offset = offsets->tseqbase;
- } else {
- /* TODO handle multiple offsets */
- assert(0);
- offset = *(oid*)Tloc(offsets, 0);
- }
-
- if (BATcount(b) >= offset+BATcount(oi)){
- BAT *ui = BATdense(0, offset, BATcount(oi));
- if (BATreplace(b, ui, oi, true) != GDK_SUCCEED) {
- if (oi != i)
- bat_destroy(oi);
- bat_destroy(b);
- bat_destroy(ui);
- return LOG_ERR;
- }
- assert(!isVIEW(b));
- bat_destroy(ui);
- } else {
- if (BATcount(b) < offset) { /* add space */
- const void *tv = ATOMnilptr(b->ttype);
- lng d = offset - BATcount(b);
- for(lng j=0;j<d;j++) {
- if (BUNappend(b, tv, true) != GDK_SUCCEED) {
- if (oi != i)
- bat_destroy(oi);
- bat_destroy(b);
- return LOG_ERR;
- }
- }
- }
- if (isVIEW(oi) && b->batCacheid == VIEWtparent(oi)) {
- BAT *ic = COLcopy(oi, oi->ttype, true, TRANSIENT);
-
- if (ic == NULL || BATappend(b, ic, NULL, true) != GDK_SUCCEED) {
- if (oi != i)
- bat_destroy(oi);
- bat_destroy(ic);
- bat_destroy(b);
- return LOG_ERR;
- }
- bat_destroy(ic);
- } else if (BATappend(b, oi, NULL, true) != GDK_SUCCEED) {
- if (oi != i)
- bat_destroy(oi);
- bat_destroy(b);
- return LOG_ERR;
- }
+ if (BATupdate(b, offsets, oi, true) != GDK_SUCCEED) {
+ if (oi != i)
+ bat_destroy(oi);
+ bat_destroy(b);
+ return LOG_ERR;
}
if (oi != i)
bat_destroy(oi);
- assert(!isVIEW(b));
bat_destroy(b);
return LOG_OK;
}
@@ -2995,8 +2948,103 @@ tc_gc_del( sql_store Store, sql_change *
}
static BAT *
+add_offsets(BAT *pos, BUN slot, size_t nr, size_t total)
+{
+ if (nr == 0)
+ return pos;
+ assert (nr > 0);
+ if (!pos && nr == total)
+ return BATdense(0, slot, nr);
+ if (!pos) {
+ pos = COLnew(0, TYPE_oid, total, TRANSIENT);
+ if (!pos)
+ return NULL;
+ }
+ for(size_t i = 0; i < nr; i++) {
+ oid v = slot + i;
+ if (BUNappend(pos, &v, true) != GDK_SUCCEED)
+ return NULL;
+ }
+ return pos;
+}
+
+static BAT *
+claim_segmentsV2(sql_trans *tr, sql_table *t, storage *s, size_t cnt)
+{
+ int in_transaction = segments_in_transaction(tr, t), ok = LOG_OK;
+ assert(s->segs);
+ ulng oldest = store_oldest(tr->store);
+ BUN slot = 0;
+ BAT *pos = NULL;
+ size_t total = cnt;
+
+ /* naive vacuum approach, iterator through segments, use deleted segments or create \
new segment at the end */ + for (segment *seg = s->segs->h, *p = NULL; seg && cnt && \
ok == LOG_OK; p = seg, seg = seg->next) { + if (seg->deleted && seg->ts < oldest && \
seg->end > seg->start) { /* re-use old deleted or rolledback append */ + if \
((seg->end - seg->start) >= cnt) { _______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic