[prev in list] [next in list] [prev in thread] [next in thread]
List: mercurial-devel
Subject: Re: [PATCH 4 of 5 flagprocessor v6] revlog: processflags()
From: Rémi_Chaintron <remi.chaintron () gmail ! com>
Date: 2016-12-30 18:01:30
Message-ID: CAB9COJG=s_W5qgVu_qx8SAirqEZ+WK-Ljd+ScJGa+O0+i7LJdw () mail ! gmail ! com
[Download RAW message or body]
[Attachment #2 (multipart/alternative)]
Updated, will send with v7.
Thanks!
On Sat, 24 Dec 2016 at 15:07 Gregory Szorc <gregory.szorc@gmail.com> wrote:
> On Sat, Dec 24, 2016 at 9:36 AM, Remi Chaintron <remi@fb.com> wrote:
>
> # HG changeset patch
> # User Remi Chaintron <remi@fb.com>
> # Date 1482596881 18000
> # Sat Dec 24 11:28:01 2016 -0500
> # Node ID c94d2907a470ca03b4a4a8da514b66d2f8952bce
> # Parent b1d220e584e6fc861a40a5aeefa0c3b19ae09126
> revlog: processflags()
>
> diff --git a/mercurial/bundlerepo.py b/mercurial/bundlerepo.py
> --- a/mercurial/bundlerepo.py
> +++ b/mercurial/bundlerepo.py
> @@ -148,7 +148,9 @@
> delta = self._chunk(chain.pop())
> text = mdiff.patches(text, [delta])
>
> - self.checkhash(text, node, rev=rev)
> + text, validatehash = self.processflags(text, self.flags(rev),
> raw=raw)
> + if validatehash is True:
> + self.checkhash(text, node, rev=rev)
> self._cache = (node, rev, text)
> return text
>
> diff --git a/mercurial/revlog.py b/mercurial/revlog.py
> --- a/mercurial/revlog.py
> +++ b/mercurial/revlog.py
> @@ -56,6 +56,10 @@
> REVIDX_ISCENSORED = (1 << 15) # revision has censor metadata, must be
> verified
> REVIDX_DEFAULT_FLAGS = 0
> REVIDX_KNOWN_FLAGS = REVIDX_ISCENSORED
> +# stable order in which flags need to be processed and their transforms
> applied
> +REVIDX_FLAGS_ORDER = [
> + REVIDX_ISCENSORED,
> +]
>
> # max size of revlog with inline data
> _maxinline = 131072
> @@ -65,6 +69,33 @@
> LookupError = error.LookupError
> CensoredNodeError = error.CensoredNodeError
>
> +# 'flagtransform' namedtuple type for extensions to use to register
> transforms
> +# on revlog flags.
> +# - The apply transform will be used as a default behavior.
> +# - The applyraw transform will be used for changegroup generation and
> debug
> +# methods.
> +flagtransform = collections.namedtuple('flagtransform', ['apply',
> 'applyraw'])
>
>
> String literals in namedtuple declarations need u'' prefixes to prevent
> Python 3 from barfing. This can likely be done as a follow-up, since Python
> 3 isn't officially supported.
>
>
> +# Store flag transforms (cf. 'addflagtransform()' to register)
> +_flagtransforms = { }
> +
> +def addflagtransform(flag, transform):
> + """Register transforms on revision data flags.
> +
> + Invariant:
> + - Flags need to be defined in REVIDX_KNOWN_FLAGS and
> REVIDX_FLAGS_ORDER.
> + - Only one transform can be registered on a specific flag.
> + - the transform must be of type 'flagtransform'
> + """
> + if not flag & REVIDX_KNOWN_FLAGS:
> + raise error.Abort(_("cannot register transform on unknown flag."))
> + if not flag in REVIDX_FLAGS_ORDER:
> + raise error.Abort(_("flag need to be defined in
> REVIDX_FLAGS_ORDER."))
> + if _flagtransforms.get(flag, None) is not None:
> + raise error.Abort(_("cannot register multiple transforms on
> flag."))
> + if not isinstance(transform, flagtransform):
> + raise error.Abort(_("invalid transform type."))
> + _flagtransforms[flag] = transform
> +
> def getoffset(q):
> return int(q >> 16)
>
> @@ -1248,7 +1279,11 @@
> bins = bins[1:]
>
> text = mdiff.patches(text, bins)
> - self.checkhash(text, node, rev=rev)
> +
> + text, validatehash = self.processflags(text, self.flags(rev),
> raw=raw)
> + if validatehash:
> + self.checkhash(text, node, rev=rev)
> +
> self._cache = (node, rev, text)
> return text
>
> @@ -1260,6 +1295,56 @@
> """
> return hash(text, p1, p2)
>
> + def processflags(self, text, flags, raw=False, reverse=False):
> + """Process revision flags and apply registered transforms.
> +
> + ``text`` - the revision data to process
> + ``flags`` - the revision flags
> + ``raw`` - an optional argument describing if the raw transform
> should be
> + applied.
> + ``reverse`` - an optional argument describing whether the flags
> should
> + be processed in order or reverse order.
> +
> + This method processes the flags in the order (or reverse order if
> + ``reverse`` is False) defined by REVIDX_FLAGS_ORDER, applying the
> + transforms registered for present flags. The order of flags
> defined in
> + REVIDX_FLAGS_ORDER needs to be stable for non-commutative
> + operations.
> +
> + If a ``flagtransform`` namedtuple has been registered on a flag,
> the
> + ``raw`` argument is used to determine which transform apply. If
> ``raw``
> + is True, the ``applyraw`` field is used, ``apply`` if not
> (default).
> + If the ``raw`` argument is set, it indicates we are currently
> processing
> + flags in the context of changegroup generation or debug commands.
> +
> + Returns a 2-tuple of ``(text, validatehash)`` where ``text`` is
> the
> + processed text and ``validatehash`` is a bool indicating whether
> the
> + returned text should be checked for hash integrity.
> + """
> + # Check all flags are known.
> + if flags & ~REVIDX_KNOWN_FLAGS:
> + raise RevlogError(_('incompatible revision flag %x') %
> + (flags & ~REVIDX_KNOWN_FLAGS))
> + validatehash = True
> + # Depending on the operation (read or write), the order might be
> + # reversed due to non-commutative transforms.
> + orderedflags = REVIDX_FLAGS_ORDER
> + if reverse:
> + orderedflags = reversed(orderedflags)
> +
> + for flag in orderedflags:
> + # If a transform has been registered for a known flag, apply
> and
> + # update result tuple.
> + if flag & flags:
> + transform = _flagtransforms.get(flag, None)
> + if transform is not None:
> + apply, applyraw = transform
> + f = apply
> + if raw:
> + f = applyraw
> + text, validatehash = f(self, text)
> + return text, validatehash
> +
> def checkhash(self, text, node, p1=None, p2=None, rev=None):
> """Check node hash integrity.
>
> @@ -1447,7 +1532,11 @@
> btext[0] = mdiff.patch(basetext, delta)
>
> try:
> - self.checkhash(btext[0], node, p1=p1, p2=p2)
> + btext[0], validatehash = self.processflags(btext[0],
> flags,
> + raw=raw,
> + reverse=True)
> + if validatehash is True:
> + self.checkhash(btext[0], node, p1=p1, p2=p2)
> if flags & REVIDX_ISCENSORED:
> raise RevlogError(_('node %s is not censored') % node)
> except CensoredNodeError:
> _______________________________________________
> Mercurial-devel mailing list
> Mercurial-devel@mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
>
> _______________________________________________
> Mercurial-devel mailing list
> Mercurial-devel@mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
>
--
Rémi
[Attachment #5 (text/html)]
<div dir="ltr">Updated, will send with v7.<div>Thanks!</div></div><br><div \
class="gmail_quote"><div dir="ltr">On Sat, 24 Dec 2016 at 15:07 Gregory Szorc <<a \
href="mailto:gregory.szorc@gmail.com">gregory.szorc@gmail.com</a>> \
wrote:<br></div><blockquote class="gmail_quote" style="margin:0 0 0 \
.8ex;border-left:1px #ccc solid;padding-left:1ex"><div dir="ltr" class="gmail_msg">On \
Sat, Dec 24, 2016 at 9:36 AM, Remi Chaintron <span dir="ltr" class="gmail_msg"><<a \
href="mailto:remi@fb.com" class="gmail_msg" \
target="_blank">remi@fb.com</a>></span> wrote:<br class="gmail_msg"></div><div \
dir="ltr" class="gmail_msg"><div class="gmail_extra gmail_msg"><div \
class="gmail_quote gmail_msg"><blockquote class="gmail_quote gmail_msg" \
style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex"># HG changeset \
patch<br class="gmail_msg"> # User Remi Chaintron <<a href="mailto:remi@fb.com" \
class="gmail_msg" target="_blank">remi@fb.com</a>><br class="gmail_msg"> # Date \
1482596881 18000<br class="gmail_msg"> # Sat Dec 24 11:28:01 2016 -0500<br \
class="gmail_msg"> # Node ID c94d2907a470ca03b4a4a8da514b66d2f8952bce<br \
class="gmail_msg"> # Parent b1d220e584e6fc861a40a5aeefa0c3b19ae09126<br \
class="gmail_msg">
revlog: processflags()<br class="gmail_msg">
<br class="gmail_msg">
diff --git a/mercurial/bundlerepo.py b/mercurial/bundlerepo.py<br class="gmail_msg">
--- a/mercurial/bundlerepo.py<br class="gmail_msg">
+++ b/mercurial/bundlerepo.py<br class="gmail_msg">
@@ -148,7 +148,9 @@<br class="gmail_msg">
delta = self._chunk(chain.pop())<br class="gmail_msg">
text = mdiff.patches(text, [delta])<br class="gmail_msg">
<br class="gmail_msg">
- self.checkhash(text, node, rev=rev)<br class="gmail_msg">
+ text, validatehash = self.processflags(text, self.flags(rev), \
raw=raw)<br class="gmail_msg"> + if validatehash is True:<br \
class="gmail_msg"> + self.checkhash(text, node, rev=rev)<br \
class="gmail_msg"> self._cache = (node, rev, text)<br class="gmail_msg">
return text<br class="gmail_msg">
<br class="gmail_msg">
diff --git a/mercurial/revlog.py b/mercurial/revlog.py<br class="gmail_msg">
--- a/mercurial/revlog.py<br class="gmail_msg">
+++ b/mercurial/revlog.py<br class="gmail_msg">
@@ -56,6 +56,10 @@<br class="gmail_msg">
REVIDX_ISCENSORED = (1 << 15) # revision has censor metadata, must be \
verified<br class="gmail_msg"> REVIDX_DEFAULT_FLAGS = 0<br class="gmail_msg">
REVIDX_KNOWN_FLAGS = REVIDX_ISCENSORED<br class="gmail_msg">
+# stable order in which flags need to be processed and their transforms applied<br \
class="gmail_msg"> +REVIDX_FLAGS_ORDER = [<br class="gmail_msg">
+ REVIDX_ISCENSORED,<br class="gmail_msg">
+]<br class="gmail_msg">
<br class="gmail_msg">
# max size of revlog with inline data<br class="gmail_msg">
_maxinline = 131072<br class="gmail_msg">
@@ -65,6 +69,33 @@<br class="gmail_msg">
LookupError = error.LookupError<br class="gmail_msg">
CensoredNodeError = error.CensoredNodeError<br class="gmail_msg">
<br class="gmail_msg">
+# 'flagtransform' namedtuple type for extensions to use to register \
transforms<br class="gmail_msg"> +# on revlog flags.<br class="gmail_msg">
+# - The apply transform will be used as a default behavior.<br class="gmail_msg">
+# - The applyraw transform will be used for changegroup generation and debug<br \
class="gmail_msg"> +# methods.<br class="gmail_msg">
+flagtransform = collections.namedtuple('flagtransform', ['apply', \
'applyraw'])<br class="gmail_msg"></blockquote><div class="gmail_msg"><br \
class="gmail_msg"></div></div></div></div><div dir="ltr" class="gmail_msg"><div \
class="gmail_extra gmail_msg"><div class="gmail_quote gmail_msg"><div \
class="gmail_msg">String literals in namedtuple declarations need u'' \
prefixes to prevent Python 3 from barfing. This can likely be done as a follow-up, \
since Python 3 isn't officially supported.<br \
class="gmail_msg"></div></div></div></div><div dir="ltr" class="gmail_msg"><div \
class="gmail_extra gmail_msg"><div class="gmail_quote gmail_msg"><div \
class="gmail_msg"> </div><blockquote class="gmail_quote gmail_msg" style="margin:0 0 \
0 .8ex;border-left:1px #ccc solid;padding-left:1ex"> +# Store flag transforms (cf. \
'addflagtransform()' to register)<br class="gmail_msg"> +_flagtransforms = { \
}<br class="gmail_msg"> +<br class="gmail_msg">
+def addflagtransform(flag, transform):<br class="gmail_msg">
+ """Register transforms on revision data flags.<br \
class="gmail_msg"> +<br class="gmail_msg">
+ Invariant:<br class="gmail_msg">
+ - Flags need to be defined in REVIDX_KNOWN_FLAGS and REVIDX_FLAGS_ORDER.<br \
class="gmail_msg"> + - Only one transform can be registered on a specific \
flag.<br class="gmail_msg"> + - the transform must be of type \
'flagtransform'<br class="gmail_msg"> + """<br \
class="gmail_msg"> + if not flag & REVIDX_KNOWN_FLAGS:<br class="gmail_msg">
+ raise error.Abort(_("cannot register transform on unknown \
flag."))<br class="gmail_msg"> + if not flag in REVIDX_FLAGS_ORDER:<br \
class="gmail_msg"> + raise error.Abort(_("flag need to be defined in \
REVIDX_FLAGS_ORDER."))<br class="gmail_msg"> + if _flagtransforms.get(flag, \
None) is not None:<br class="gmail_msg"> + raise \
error.Abort(_("cannot register multiple transforms on flag."))<br \
class="gmail_msg"> + if not isinstance(transform, flagtransform):<br \
class="gmail_msg"> + raise error.Abort(_("invalid transform \
type."))<br class="gmail_msg"> + _flagtransforms[flag] = transform<br \
class="gmail_msg"> +<br class="gmail_msg">
def getoffset(q):<br class="gmail_msg">
return int(q >> 16)<br class="gmail_msg">
<br class="gmail_msg">
@@ -1248,7 +1279,11 @@<br class="gmail_msg">
bins = bins[1:]<br class="gmail_msg">
<br class="gmail_msg">
text = mdiff.patches(text, bins)<br class="gmail_msg">
- self.checkhash(text, node, rev=rev)<br class="gmail_msg">
+<br class="gmail_msg">
+ text, validatehash = self.processflags(text, self.flags(rev), \
raw=raw)<br class="gmail_msg"> + if validatehash:<br class="gmail_msg">
+ self.checkhash(text, node, rev=rev)<br class="gmail_msg">
+<br class="gmail_msg">
self._cache = (node, rev, text)<br class="gmail_msg">
return text<br class="gmail_msg">
<br class="gmail_msg">
@@ -1260,6 +1295,56 @@<br class="gmail_msg">
"""<br class="gmail_msg">
return hash(text, p1, p2)<br class="gmail_msg">
<br class="gmail_msg">
+ def processflags(self, text, flags, raw=False, reverse=False):<br \
class="gmail_msg"> + """Process revision flags and apply \
registered transforms.<br class="gmail_msg"> +<br class="gmail_msg">
+ ``text`` - the revision data to process<br class="gmail_msg">
+ ``flags`` - the revision flags<br class="gmail_msg">
+ ``raw`` - an optional argument describing if the raw transform should \
be<br class="gmail_msg"> + applied.<br class="gmail_msg">
+ ``reverse`` - an optional argument describing whether the flags \
should<br class="gmail_msg"> + be processed in order or reverse order.<br \
class="gmail_msg"> +<br class="gmail_msg">
+ This method processes the flags in the order (or reverse order if<br \
class="gmail_msg"> + ``reverse`` is False) defined by REVIDX_FLAGS_ORDER, \
applying the<br class="gmail_msg"> + transforms registered for present \
flags. The order of flags defined in<br class="gmail_msg"> + \
REVIDX_FLAGS_ORDER needs to be stable for non-commutative<br class="gmail_msg"> + \
operations.<br class="gmail_msg"> +<br class="gmail_msg">
+ If a ``flagtransform`` namedtuple has been registered on a flag, the<br \
class="gmail_msg"> + ``raw`` argument is used to determine which transform \
apply. If ``raw``<br class="gmail_msg"> + is True, the ``applyraw`` field \
is used, ``apply`` if not (default).<br class="gmail_msg"> + If the \
``raw`` argument is set, it indicates we are currently processing<br \
class="gmail_msg"> + flags in the context of changegroup generation or \
debug commands.<br class="gmail_msg"> +<br class="gmail_msg">
+ Returns a 2-tuple of ``(text, validatehash)`` where ``text`` is the<br \
class="gmail_msg"> + processed text and ``validatehash`` is a bool \
indicating whether the<br class="gmail_msg"> + returned text should be \
checked for hash integrity.<br class="gmail_msg"> + """<br \
class="gmail_msg"> + # Check all flags are known.<br class="gmail_msg">
+ if flags & ~REVIDX_KNOWN_FLAGS:<br class="gmail_msg">
+ raise RevlogError(_('incompatible revision flag %x') %<br \
class="gmail_msg"> + (flags & \
~REVIDX_KNOWN_FLAGS))<br class="gmail_msg"> + validatehash = True<br \
class="gmail_msg"> + # Depending on the operation (read or write), the \
order might be<br class="gmail_msg"> + # reversed due to non-commutative \
transforms.<br class="gmail_msg"> + orderedflags = REVIDX_FLAGS_ORDER<br \
class="gmail_msg"> + if reverse:<br class="gmail_msg">
+ orderedflags = reversed(orderedflags)<br class="gmail_msg">
+<br class="gmail_msg">
+ for flag in orderedflags:<br class="gmail_msg">
+ # If a transform has been registered for a known flag, apply \
and<br class="gmail_msg"> + # update result tuple.<br \
class="gmail_msg"> + if flag & flags:<br class="gmail_msg">
+ transform = _flagtransforms.get(flag, None)<br \
class="gmail_msg"> + if transform is not None:<br \
class="gmail_msg"> + apply, applyraw = transform<br \
class="gmail_msg"> + f = apply<br class="gmail_msg">
+ if raw:<br class="gmail_msg">
+ f = applyraw<br class="gmail_msg">
+ text, validatehash = f(self, text)<br \
class="gmail_msg"> + return text, validatehash<br class="gmail_msg">
+<br class="gmail_msg">
def checkhash(self, text, node, p1=None, p2=None, rev=None):<br \
class="gmail_msg">
"""Check node hash integrity.<br class="gmail_msg">
<br class="gmail_msg">
@@ -1447,7 +1532,11 @@<br class="gmail_msg">
btext[0] = mdiff.patch(basetext, delta)<br \
class="gmail_msg"> <br class="gmail_msg">
try:<br class="gmail_msg">
- self.checkhash(btext[0], node, p1=p1, p2=p2)<br \
class="gmail_msg"> + btext[0], validatehash = \
self.processflags(btext[0], flags,<br class="gmail_msg"> + \
raw=raw,<br class="gmail_msg"> + \
reverse=True)<br class="gmail_msg"> + if validatehash is \
True:<br class="gmail_msg"> + self.checkhash(btext[0], \
node, p1=p1, p2=p2)<br class="gmail_msg">
if flags & REVIDX_ISCENSORED:<br class="gmail_msg">
raise RevlogError(_('node %s is not \
censored') % node)<br class="gmail_msg"> except CensoredNodeError:<br \
class="gmail_msg"> _______________________________________________<br \
class="gmail_msg"> Mercurial-devel mailing list<br class="gmail_msg">
<a href="mailto:Mercurial-devel@mercurial-scm.org" class="gmail_msg" \
target="_blank">Mercurial-devel@mercurial-scm.org</a><br class="gmail_msg"> <a \
href="https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel" \
rel="noreferrer" class="gmail_msg" \
target="_blank">https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel</a><br \
class="gmail_msg"> </blockquote></div></div></div>
_______________________________________________<br class="gmail_msg">
Mercurial-devel mailing list<br class="gmail_msg">
<a href="mailto:Mercurial-devel@mercurial-scm.org" class="gmail_msg" \
target="_blank">Mercurial-devel@mercurial-scm.org</a><br class="gmail_msg"> <a \
href="https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel" \
rel="noreferrer" class="gmail_msg" \
target="_blank">https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel</a><br \
class="gmail_msg"> </blockquote></div><div dir="ltr">-- <br></div><div \
data-smartmail="gmail_signature"><div dir="ltr">Rémi</div></div>
[Attachment #6 (text/plain)]
_______________________________________________
Mercurial-devel mailing list
Mercurial-devel@mercurial-scm.org
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic