[prev in list] [next in list] [prev in thread] [next in thread]
List: pypy-svn
Subject: [pypy-commit] pypy py3.5: merge default into branch
From: mattip <pypy.commits () gmail ! com>
Date: 2018-12-23 14:57:14
Message-ID: 5c1fa24a.1c69fb81.97f39.098b () mx ! google ! com
[Download RAW message or body]
Author: Matti Picus <matti.picus@gmail.com>
Branch: py3.5
Changeset: r95520:406030262749
Date: 2018-12-23 16:53 +0200
http://bitbucket.org/pypy/pypy/changeset/406030262749/
Log: merge default into branch
diff --git a/extra_tests/cffi_tests/cffi0/test_ffi_backend.py \
b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py
--- a/extra_tests/cffi_tests/cffi0/test_ffi_backend.py
+++ b/extra_tests/cffi_tests/cffi0/test_ffi_backend.py
@@ -327,6 +327,16 @@
assert ffi.typeof(c) is ffi.typeof("char[]")
ffi.cast("unsigned short *", c)[1] += 500
assert list(a) == [10000, 20500, 30000]
+ assert c == ffi.from_buffer(a, True)
+ assert c == ffi.from_buffer(a, require_writable=True)
+ #
+ p = ffi.from_buffer(b"abcd")
+ assert p[2] == b"c"
+ #
+ assert p == ffi.from_buffer(b"abcd", False)
+ py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True)
+ py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd",
+ require_writable=True)
def test_memmove(self):
ffi = FFI()
diff --git a/extra_tests/cffi_tests/cffi1/test_ffi_obj.py \
b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py
--- a/extra_tests/cffi_tests/cffi1/test_ffi_obj.py
+++ b/extra_tests/cffi_tests/cffi1/test_ffi_obj.py
@@ -244,6 +244,16 @@
assert ffi.typeof(c) is ffi.typeof("char[]")
ffi.cast("unsigned short *", c)[1] += 500
assert list(a) == [10000, 20500, 30000]
+ assert c == ffi.from_buffer(a, True)
+ assert c == ffi.from_buffer(a, require_writable=True)
+ #
+ p = ffi.from_buffer(b"abcd")
+ assert p[2] == b"c"
+ #
+ assert p == ffi.from_buffer(b"abcd", False)
+ py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True)
+ py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd",
+ require_writable=True)
def test_memmove():
ffi = _cffi1_backend.FFI()
diff --git a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py \
b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
--- a/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/extra_tests/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1654,6 +1654,16 @@
assert ffi.typeof(c) is ffi.typeof("char[]")
ffi.cast("unsigned short *", c)[1] += 500
assert list(a) == [10000, 20500, 30000]
+ assert c == ffi.from_buffer(a, True)
+ assert c == ffi.from_buffer(a, require_writable=True)
+ #
+ p = ffi.from_buffer(b"abcd")
+ assert p[2] == b"c"
+ #
+ assert p == ffi.from_buffer(b"abcd", False)
+ py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True)
+ py.test.raises((TypeError, BufferError), ffi.from_buffer, b"abcd",
+ require_writable=True)
def test_all_primitives(self):
assert set(PRIMITIVE_TO_INDEX) == set([
diff --git a/extra_tests/test_pyrepl/conftest.py \
b/extra_tests/test_pyrepl/conftest.py new file mode 100644
--- /dev/null
+++ b/extra_tests/test_pyrepl/conftest.py
@@ -0,0 +1,8 @@
+import sys
+
+def pytest_ignore_collect(path):
+ if '__pypy__' not in sys.builtin_module_names:
+ try:
+ import pyrepl
+ except ImportError:
+ return True
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -341,7 +341,7 @@
# """
# note that 'buffer' is a type, set on this instance by __init__
- def from_buffer(self, python_buffer):
+ def from_buffer(self, python_buffer, require_writable=False):
"""Return a <cdata 'char[]'> that points to the data of the
given Python object, which must support the buffer interface.
Note that this is not meant to be used on the built-in types
@@ -349,7 +349,8 @@
but only on objects containing large quantities of raw data
in some other format, like 'array.array' or numpy arrays.
"""
- return self._backend.from_buffer(self.BCharA, python_buffer)
+ return self._backend.from_buffer(self.BCharA, python_buffer,
+ require_writable)
def memmove(self, dest, src, n):
"""ffi.memmove(dest, src, n) copies n bytes of memory from src to dest.
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -401,8 +401,10 @@
* some functions and attributes of the ``gc`` module behave in a
slightly different way: for example, ``gc.enable`` and
- ``gc.disable`` are supported, but instead of enabling and disabling
- the GC, they just enable and disable the execution of finalizers.
+ ``gc.disable`` are supported, but "enabling and disabling the GC" has
+ a different meaning in PyPy than in CPython. These functions
+ actually enable and disable the major collections and the
+ execution of finalizers.
* PyPy prints a random line from past #pypy IRC topics at startup in
interactive mode. In a released version, this behaviour is suppressed, but
diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst
--- a/pypy/doc/gc_info.rst
+++ b/pypy/doc/gc_info.rst
@@ -22,8 +22,44 @@
larger. (A third category, the very large objects, are initially allocated
outside the nursery and never move.)
-Since Incminimark is an incremental GC, the major collection is incremental,
-meaning there should not be any pauses longer than 1ms.
+Since Incminimark is an incremental GC, the major collection is incremental:
+the goal is not to have any pause longer than 1ms, but in practice it depends
+on the size and characteristics of the heap: occasionally, there can be pauses
+between 10-100ms.
+
+
+Semi-manual GC management
+--------------------------
+
+If there are parts of the program where it is important to have a low latency,
+you might want to control precisely when the GC runs, to avoid unexpected
+pauses. Note that this has effect only on major collections, while minor
+collections continue to work as usual.
+
+As explained above, a full major collection consists of ``N`` steps, where
+``N`` depends on the size of the heap; generally speaking, it is not possible
+to predict how many steps will be needed to complete a collection.
+
+``gc.enable()`` and ``gc.disable()`` control whether the GC runs collection
+steps automatically. When the GC is disabled the memory usage will grow
+indefinitely, unless you manually call ``gc.collect()`` and
+``gc.collect_step()``.
+
+``gc.collect()`` runs a full major collection.
+
+``gc.collect_step()`` runs a single collection step. It returns an object of
+type GcCollectStepStats_, the same which is passed to the corresponding `GC
+Hooks`_. The following code is roughly equivalent to a ``gc.collect()``::
+
+ while True:
+ if gc.collect_step().major_is_done:
+ break
+
+For a real-world example of usage of this API, you can look at the 3rd-party
+module `pypytools.gc.custom`_, which also provides a ``with customgc.nogc()``
+context manager to mark sections where the GC is forbidden.
+
+.. _`pypytools.gc.custom`: \
https://bitbucket.org/antocuni/pypytools/src/0273afc3e8bedf0eb1ef630c3bc69e8d9dd661fe/pypytools/gc/custom.py?at=default&fileviewer=file-view-default
Fragmentation
@@ -184,6 +220,8 @@
the number of pinned objects.
+.. _GcCollectStepStats:
+
The attributes for ``GcCollectStepStats`` are:
``count``, ``duration``, ``duration_min``, ``duration_max``
@@ -192,10 +230,14 @@
``oldstate``, ``newstate``
Integers which indicate the state of the GC before and after the step.
+``major_is_done``
+ Boolean which indicate whether this was the last step of the major
+ collection
+
The value of ``oldstate`` and ``newstate`` is one of these constants, defined
inside ``gc.GcCollectStepStats``: ``STATE_SCANNING``, ``STATE_MARKING``,
-``STATE_SWEEPING``, ``STATE_FINALIZING``. It is possible to get a string
-representation of it by indexing the ``GC_STATS`` tuple.
+``STATE_SWEEPING``, ``STATE_FINALIZING``, ``STATE_USERDEL``. It is possible
+to get a string representation of it by indexing the ``GC_STATES`` tuple.
The attributes for ``GcCollectStats`` are:
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -60,3 +60,10 @@
.. branch: cleanup-test_lib_pypy
Update most test_lib_pypy/ tests and move them to extra_tests/.
+
+.. branch: gc-disable
+
+Make it possible to manually manage the GC by using a combination of
+gc.disable() and gc.collect_step(). Make sure to write a proper release
+announcement in which we explain that existing programs could leak memory if
+they run for too much time between a gc.disable()/gc.enable()
diff --git a/pypy/module/_cffi_backend/ffi_obj.py \
b/pypy/module/_cffi_backend/ffi_obj.py
--- a/pypy/module/_cffi_backend/ffi_obj.py
+++ b/pypy/module/_cffi_backend/ffi_obj.py
@@ -327,7 +327,8 @@
return w_ctype.cast(w_ob)
- def descr_from_buffer(self, w_python_buffer):
+ @unwrap_spec(require_writable=int)
+ def descr_from_buffer(self, w_python_buffer, require_writable=0):
"""\
Return a <cdata 'char[]'> that points to the data of the given Python
object, which must support the buffer interface. Note that this is
@@ -337,7 +338,8 @@
'array.array' or numpy arrays."""
#
w_ctchara = newtype._new_chara_type(self.space)
- return func._from_buffer(self.space, w_ctchara, w_python_buffer)
+ return func._from_buffer(self.space, w_ctchara, w_python_buffer,
+ require_writable)
@unwrap_spec(w_arg=W_CData)
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -110,8 +110,8 @@
def _fetch_as_write_buffer(space, w_x):
return space.writebuf_w(w_x)
-@unwrap_spec(w_ctype=ctypeobj.W_CType)
-def from_buffer(space, w_ctype, w_x):
+@unwrap_spec(w_ctype=ctypeobj.W_CType, require_writable=int)
+def from_buffer(space, w_ctype, w_x, require_writable=0):
from pypy.module._cffi_backend import ctypearray, ctypeprim
#
if (not isinstance(w_ctype, ctypearray.W_CTypeArray) or
@@ -119,13 +119,16 @@
raise oefmt(space.w_TypeError,
"needs 'char[]', got '%s'", w_ctype.name)
#
- return _from_buffer(space, w_ctype, w_x)
+ return _from_buffer(space, w_ctype, w_x, require_writable)
-def _from_buffer(space, w_ctype, w_x):
+def _from_buffer(space, w_ctype, w_x, require_writable):
if space.isinstance_w(w_x, space.w_unicode):
raise oefmt(space.w_TypeError,
- "from_buffer() cannot return the address a unicode")
- buf = _fetch_as_read_buffer(space, w_x)
+ "from_buffer() cannot return the address of a unicode object")
+ if require_writable:
+ buf = _fetch_as_write_buffer(space, w_x)
+ else:
+ buf = _fetch_as_read_buffer(space, w_x)
if space.isinstance_w(w_x, space.w_bytes):
_cdata = get_raw_address_of_string(space, w_x)
else:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py \
b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3730,6 +3730,18 @@
check(4 | 8, "CHB", "GTB")
check(4 | 16, "CHB", "ROB")
+def test_from_buffer_require_writable():
+ BChar = new_primitive_type("char")
+ BCharP = new_pointer_type(BChar)
+ BCharA = new_array_type(BCharP, None)
+ p1 = from_buffer(BCharA, b"foo", False)
+ assert p1 == from_buffer(BCharA, b"foo", False)
+ py.test.raises((TypeError, BufferError), from_buffer, BCharA, b"foo", True)
+ ba = bytearray(b"foo")
+ p1 = from_buffer(BCharA, ba, True)
+ p1[0] = b"g"
+ assert ba == b"goo"
+
def test_memmove():
Short = new_primitive_type("short")
ShortA = new_array_type(new_pointer_type(Short), None)
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py \
b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -287,6 +287,16 @@
assert ffi.typeof(c) is ffi.typeof("char[]")
ffi.cast("unsigned short *", c)[1] += 500
assert list(a) == [10000, 20500, 30000]
+ assert c == ffi.from_buffer(a, True)
+ assert c == ffi.from_buffer(a, require_writable=True)
+ #
+ p = ffi.from_buffer(b"abcd")
+ assert p[2] == b"c"
+ #
+ assert p == ffi.from_buffer(b"abcd", False)
+ raises((TypeError, BufferError), ffi.from_buffer, b"abcd", True)
+ raises((TypeError, BufferError), ffi.from_buffer, b"abcd",
+ require_writable=True)
def test_from_buffer_BytesIO(self):
from _cffi_backend import FFI
diff --git a/pypy/module/_cppyy/test/Makefile b/pypy/module/_cppyy/test/Makefile
--- a/pypy/module/_cppyy/test/Makefile
+++ b/pypy/module/_cppyy/test/Makefile
@@ -15,7 +15,7 @@
HASGENREFLEX:=$(shell command -v genreflex 2> /dev/null)
-cppflags=-std=c++14 -O3 -m64 -fPIC -rdynamic
+cppflags=-std=c++14 -O3 -fPIC -rdynamic
ifdef HASGENREFLEX
genreflex_flags:=$(shell genreflex --cppflags)
cppflags+=$(genreflex_flags)
@@ -25,7 +25,7 @@
PLATFORM := $(shell uname -s)
ifeq ($(PLATFORM),Darwin)
- cppflags+=-dynamiclib -single_module -arch x86_64 -undefined dynamic_lookup
+ cppflags+=-dynamiclib -single_module -undefined dynamic_lookup
endif
diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py
--- a/pypy/module/gc/__init__.py
+++ b/pypy/module/gc/__init__.py
@@ -4,6 +4,7 @@
class Module(MixedModule):
interpleveldefs = {
'collect': 'interp_gc.collect',
+ 'collect_step': 'interp_gc.collect_step',
'enable': 'interp_gc.enable',
'disable': 'interp_gc.disable',
'isenabled': 'interp_gc.isenabled',
diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py
--- a/pypy/module/gc/hook.py
+++ b/pypy/module/gc/hook.py
@@ -1,5 +1,6 @@
from rpython.memory.gc.hook import GcHooks
-from rpython.memory.gc import incminimark
+from rpython.memory.gc import incminimark
+from rpython.rlib import rgc
from rpython.rlib.nonconst import NonConstant
from rpython.rlib.rarithmetic import r_uint, r_longlong, longlongmax
from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
@@ -117,12 +118,24 @@
self.descr_set_on_gc_collect(space, space.w_None)
-class GcMinorHookAction(AsyncAction):
+class NoRecursiveAction(AsyncAction):
+ depth = 0
+
+ def perform(self, ec, frame):
+ if self.depth == 0:
+ try:
+ self.depth += 1
+ return self._do_perform(ec, frame)
+ finally:
+ self.depth -= 1
+
+
+class GcMinorHookAction(NoRecursiveAction):
total_memory_used = 0
pinned_objects = 0
def __init__(self, space):
- AsyncAction.__init__(self, space)
+ NoRecursiveAction.__init__(self, space)
self.w_callable = space.w_None
self.reset()
@@ -145,7 +158,7 @@
self.pinned_objects = NonConstant(-42)
self.fire()
- def perform(self, ec, frame):
+ def _do_perform(self, ec, frame):
w_stats = W_GcMinorStats(
self.count,
self.duration,
@@ -157,12 +170,12 @@
self.space.call_function(self.w_callable, w_stats)
-class GcCollectStepHookAction(AsyncAction):
+class GcCollectStepHookAction(NoRecursiveAction):
oldstate = 0
newstate = 0
def __init__(self, space):
- AsyncAction.__init__(self, space)
+ NoRecursiveAction.__init__(self, space)
self.w_callable = space.w_None
self.reset()
@@ -185,19 +198,20 @@
self.newstate = NonConstant(-42)
self.fire()
- def perform(self, ec, frame):
+ def _do_perform(self, ec, frame):
w_stats = W_GcCollectStepStats(
self.count,
self.duration,
self.duration_min,
self.duration_max,
self.oldstate,
- self.newstate)
+ self.newstate,
+ rgc.is_done__states(self.oldstate, self.newstate))
self.reset()
self.space.call_function(self.w_callable, w_stats)
-class GcCollectHookAction(AsyncAction):
+class GcCollectHookAction(NoRecursiveAction):
num_major_collects = 0
arenas_count_before = 0
arenas_count_after = 0
@@ -206,7 +220,7 @@
rawmalloc_bytes_after = 0
def __init__(self, space):
- AsyncAction.__init__(self, space)
+ NoRecursiveAction.__init__(self, space)
self.w_callable = space.w_None
self.reset()
@@ -227,7 +241,7 @@
self.rawmalloc_bytes_after = NonConstant(r_uint(42))
self.fire()
- def perform(self, ec, frame):
+ def _do_perform(self, ec, frame):
w_stats = W_GcCollectStats(self.count,
self.num_major_collects,
self.arenas_count_before,
@@ -252,15 +266,32 @@
class W_GcCollectStepStats(W_Root):
+ # NOTE: this is specific to incminimark: if we want to integrate the
+ # applevel gc module with another gc, we probably need a more general
+ # approach to this.
+ #
+ # incminimark has 4 GC states: scanning, marking, sweeping and
+ # finalizing. However, from the user point of view, we have an additional
+ # "virtual" state: USERDEL, which represent when we run applevel
+ # finalizers after having completed a GC major collection. This state is
+ # never explicitly visible when using hooks, but it is used for the return
+ # value of gc.collect_step (see interp_gc.py)
+ STATE_SCANNING = incminimark.STATE_SCANNING
+ STATE_MARKING = incminimark.STATE_MARKING
+ STATE_SWEEPING = incminimark.STATE_SWEEPING
+ STATE_FINALIZING = incminimark.STATE_FINALIZING
+ STATE_USERDEL = incminimark.STATE_FINALIZING + 1 # used by StepCollector
+ GC_STATES = tuple(incminimark.GC_STATES + ['USERDEL'])
def __init__(self, count, duration, duration_min, duration_max,
- oldstate, newstate):
+ oldstate, newstate, major_is_done):
self.count = count
self.duration = duration
self.duration_min = duration_min
self.duration_max = duration_max
self.oldstate = oldstate
self.newstate = newstate
+ self.major_is_done = major_is_done
class W_GcCollectStats(W_Root):
@@ -320,11 +351,16 @@
W_GcCollectStepStats.typedef = TypeDef(
"GcCollectStepStats",
- STATE_SCANNING = incminimark.STATE_SCANNING,
- STATE_MARKING = incminimark.STATE_MARKING,
- STATE_SWEEPING = incminimark.STATE_SWEEPING,
- STATE_FINALIZING = incminimark.STATE_FINALIZING,
- GC_STATES = tuple(incminimark.GC_STATES),
+ STATE_SCANNING = W_GcCollectStepStats.STATE_SCANNING,
+ STATE_MARKING = W_GcCollectStepStats.STATE_MARKING,
+ STATE_SWEEPING = W_GcCollectStepStats.STATE_SWEEPING,
+ STATE_FINALIZING = W_GcCollectStepStats.STATE_FINALIZING,
+ STATE_USERDEL = W_GcCollectStepStats.STATE_USERDEL,
+ GC_STATES = tuple(W_GcCollectStepStats.GC_STATES),
+ major_is_done = interp_attrproperty(
+ "major_is_done",
+ cls=W_GcCollectStepStats,
+ wrapfn="newbool"),
**wrap_many(W_GcCollectStepStats, (
"count",
"duration",
diff --git a/pypy/module/gc/interp_gc.py b/pypy/module/gc/interp_gc.py
--- a/pypy/module/gc/interp_gc.py
+++ b/pypy/module/gc/interp_gc.py
@@ -1,6 +1,7 @@
from pypy.interpreter.gateway import unwrap_spec
from pypy.interpreter.error import oefmt
from rpython.rlib import rgc
+from pypy.module.gc.hook import W_GcCollectStepStats
@unwrap_spec(generation=int)
@@ -16,7 +17,9 @@
cache.clear()
rgc.collect()
+ _run_finalizers(space)
+def _run_finalizers(space):
# if we are running in gc.disable() mode but gc.collect() is called,
# we should still call the finalizers now. We do this as an attempt
# to get closer to CPython's behavior: in Py3.5 some tests
@@ -39,18 +42,20 @@
return space.newint(0)
def enable(space):
- """Non-recursive version. Enable finalizers now.
+ """Non-recursive version. Enable major collections and finalizers.
If they were already enabled, no-op.
If they were disabled even several times, enable them anyway.
"""
+ rgc.enable()
if not space.user_del_action.enabled_at_app_level:
space.user_del_action.enabled_at_app_level = True
enable_finalizers(space)
def disable(space):
- """Non-recursive version. Disable finalizers now. Several calls
- to this function are ignored.
+ """Non-recursive version. Disable major collections and finalizers.
+ Multiple calls to this function are ignored.
"""
+ rgc.disable()
if space.user_del_action.enabled_at_app_level:
space.user_del_action.enabled_at_app_level = False
disable_finalizers(space)
@@ -77,6 +82,59 @@
if uda.pending_with_disabled_del is None:
uda.pending_with_disabled_del = []
+
+class StepCollector(object):
+ """
+ Invoke rgc.collect_step() until we are done, then run the app-level
+ finalizers as a separate step
+ """
+
+ def __init__(self, space):
+ self.space = space
+ self.finalizing = False
+
+ def do(self):
+ if self.finalizing:
+ self._run_finalizers()
+ self.finalizing = False
+ oldstate = W_GcCollectStepStats.STATE_USERDEL
+ newstate = W_GcCollectStepStats.STATE_SCANNING
+ major_is_done = True # now we are finally done
+ else:
+ states = self._collect_step()
+ oldstate = rgc.old_state(states)
+ newstate = rgc.new_state(states)
+ major_is_done = False # USERDEL still to do
+ if rgc.is_done(states):
+ newstate = W_GcCollectStepStats.STATE_USERDEL
+ self.finalizing = True
+ #
+ duration = -1
+ return W_GcCollectStepStats(
+ count = 1,
+ duration = duration,
+ duration_min = duration,
+ duration_max = duration,
+ oldstate = oldstate,
+ newstate = newstate,
+ major_is_done = major_is_done)
+
+ def _collect_step(self):
+ return rgc.collect_step()
+
+ def _run_finalizers(self):
+ _run_finalizers(self.space)
+
+def collect_step(space):
+ """
+ If the GC is incremental, run a single gc-collect-step. Return True when
+ the major collection is completed.
+ If the GC is not incremental, do a full collection and return True.
+ """
+ sc = space.fromcache(StepCollector)
+ w_stats = sc.do()
+ return w_stats
+
# ____________________________________________________________
@unwrap_spec(filename='fsencode')
diff --git a/pypy/module/gc/test/test_gc.py b/pypy/module/gc/test/test_gc.py
--- a/pypy/module/gc/test/test_gc.py
+++ b/pypy/module/gc/test/test_gc.py
@@ -1,7 +1,20 @@
import py
-
+import pytest
+from rpython.rlib import rgc
+from pypy.interpreter.baseobjspace import ObjSpace
+from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.module.gc.interp_gc import StepCollector, W_GcCollectStepStats
class AppTestGC(object):
+
+ def setup_class(cls):
+ if cls.runappdirect:
+ pytest.skip("these tests cannot work with -A")
+ space = cls.space
+ def rgc_isenabled(space):
+ return space.newbool(rgc.isenabled())
+ cls.w_rgc_isenabled = space.wrap(interp2app(rgc_isenabled))
+
def test_collect(self):
import gc
gc.collect() # mostly a "does not crash" kind of test
@@ -63,12 +76,16 @@
def test_enable(self):
import gc
assert gc.isenabled()
+ assert self.rgc_isenabled()
gc.disable()
assert not gc.isenabled()
+ assert not self.rgc_isenabled()
gc.enable()
assert gc.isenabled()
+ assert self.rgc_isenabled()
gc.enable()
assert gc.isenabled()
+ assert self.rgc_isenabled()
def test_gc_collect_overrides_gc_disable(self):
import gc
@@ -83,6 +100,24 @@
assert deleted == [1]
gc.enable()
+ def test_gc_collect_step(self):
+ import gc
+
+ class X(object):
+ deleted = 0
+ def __del__(self):
+ X.deleted += 1
+
+ gc.disable()
+ X(); X(); X();
+ n = 0
+ while True:
+ n += 1
+ if gc.collect_step().major_is_done:
+ break
+
+ assert n >= 2 # at least one step + 1 finalizing
+ assert X.deleted == 3
class AppTestGcDumpHeap(object):
pytestmark = py.test.mark.xfail(run=False)
@@ -156,3 +191,55 @@
gc.collect() # the classes C should all go away here
for r in rlist:
assert r() is None
+
+
+def test_StepCollector():
+ W = W_GcCollectStepStats
+ SCANNING = W.STATE_SCANNING
+ MARKING = W.STATE_MARKING
+ SWEEPING = W.STATE_SWEEPING
+ FINALIZING = W.STATE_FINALIZING
+ USERDEL = W.STATE_USERDEL
+
+ class MyStepCollector(StepCollector):
+ my_steps = 0
+ my_done = False
+ my_finalized = 0
+
+ def __init__(self):
+ StepCollector.__init__(self, space=None)
+ self._state_transitions = iter([
+ (SCANNING, MARKING),
+ (MARKING, SWEEPING),
+ (SWEEPING, FINALIZING),
+ (FINALIZING, SCANNING)])
+
+ def _collect_step(self):
+ self.my_steps += 1
+ try:
+ oldstate, newstate = next(self._state_transitions)
+ except StopIteration:
+ assert False, 'should not happen, did you call _collect_step too \
much?' + return rgc._encode_states(oldstate, newstate)
+
+ def _run_finalizers(self):
+ self.my_finalized += 1
+
+ sc = MyStepCollector()
+ transitions = []
+ while True:
+ result = sc.do()
+ transitions.append((result.oldstate, result.newstate, sc.my_finalized))
+ if result.major_is_done:
+ break
+
+ assert transitions == [
+ (SCANNING, MARKING, False),
+ (MARKING, SWEEPING, False),
+ (SWEEPING, FINALIZING, False),
+ (FINALIZING, USERDEL, False),
+ (USERDEL, SCANNING, True)
+ ]
+ # there is one more transition than actual step, because
+ # FINALIZING->USERDEL is "virtual"
+ assert sc.my_steps == len(transitions) - 1
diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py
--- a/pypy/module/gc/test/test_hook.py
+++ b/pypy/module/gc/test/test_hook.py
@@ -69,26 +69,29 @@
def test_on_gc_collect_step(self):
import gc
+ SCANNING = 0
+ MARKING = 1
+ SWEEPING = 2
+ FINALIZING = 3
lst = []
def on_gc_collect_step(stats):
lst.append((stats.count,
stats.duration,
stats.oldstate,
- stats.newstate))
+ stats.newstate,
+ stats.major_is_done))
gc.hooks.on_gc_collect_step = on_gc_collect_step
- self.fire_gc_collect_step(10, 20, 30)
- self.fire_gc_collect_step(40, 50, 60)
+ self.fire_gc_collect_step(10, SCANNING, MARKING)
+ self.fire_gc_collect_step(40, FINALIZING, SCANNING)
assert lst == [
- (1, 10, 20, 30),
- (1, 40, 50, 60),
+ (1, 10, SCANNING, MARKING, False),
+ (1, 40, FINALIZING, SCANNING, True),
]
#
gc.hooks.on_gc_collect_step = None
- self.fire_gc_collect_step(70, 80, 90) # won't fire
- assert lst == [
- (1, 10, 20, 30),
- (1, 40, 50, 60),
- ]
+ oldlst = lst[:]
+ self.fire_gc_collect_step(70, SCANNING, MARKING) # won't fire
+ assert lst == oldlst
def test_on_gc_collect(self):
import gc
@@ -123,7 +126,8 @@
assert S.STATE_MARKING == 1
assert S.STATE_SWEEPING == 2
assert S.STATE_FINALIZING == 3
- assert S.GC_STATES == ('SCANNING', 'MARKING', 'SWEEPING', 'FINALIZING')
+ assert S.GC_STATES == ('SCANNING', 'MARKING', 'SWEEPING',
+ 'FINALIZING', 'USERDEL')
def test_cumulative(self):
import gc
@@ -176,3 +180,22 @@
assert gc.hooks.on_gc_minor is None
assert gc.hooks.on_gc_collect_step is None
assert gc.hooks.on_gc_collect is None
+
+ def test_no_recursive(self):
+ import gc
+ lst = []
+ def on_gc_minor(stats):
+ lst.append((stats.count,
+ stats.duration,
+ stats.total_memory_used,
+ stats.pinned_objects))
+ self.fire_gc_minor(1, 2, 3) # won't fire NOW
+ gc.hooks.on_gc_minor = on_gc_minor
+ self.fire_gc_minor(10, 20, 30)
+ self.fire_gc_minor(40, 50, 60)
+ # the duration for the 2nd call is 41, because it also counts the 1
+ # which was fired recursively
+ assert lst == [
+ (1, 10, 20, 30),
+ (2, 41, 50, 60),
+ ]
diff --git a/pypy/module/math/test/test_direct.py \
b/pypy/module/math/test/test_direct.py
--- a/pypy/module/math/test/test_direct.py
+++ b/pypy/module/math/test/test_direct.py
@@ -6,11 +6,6 @@
from rpython.rtyper.lltypesystem.module.test.math_cases import (MathTests,
get_tester)
-consistent_host = True
-if '__pypy__' not in sys.builtin_module_names:
- if sys.version_info < (2, 6):
- consistent_host = False
-
class TestDirect(MathTests):
pass
@@ -30,8 +25,6 @@
def make_test_case((fnname, args, expected), dict):
#
def test_func(self):
- if not consistent_host:
- py.test.skip("inconsistent behavior before 2.6")
try:
fn = getattr(math, fnname)
except AttributeError:
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -18,7 +18,6 @@
filename = filename[:-1]
space = cls.space
cls.w_math_cases = space.wrap(filename)
- cls.w_consistent_host = space.wrap(test_direct.consistent_host)
@classmethod
def make_callable_wrapper(cls, func):
@@ -53,8 +52,6 @@
yield fnname, args, expected
def test_all_cases(self):
- if not self.consistent_host:
- skip("please test this on top of PyPy or CPython >= 2.6")
import math
for fnname, args, expected in self.cases():
fn = getattr(math, fnname)
diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -149,6 +149,20 @@
def get_size_incl_hash(self, obj):
return self.get_size(obj)
+ # these can be overriden by subclasses, called by the GCTransformer
+ def enable(self):
+ pass
+
+ def disable(self):
+ pass
+
+ def isenabled(self):
+ return True
+
+ def collect_step(self):
+ self.collect()
+ return True
+
def malloc(self, typeid, length=0, zero=False):
"""NOT_RPYTHON
For testing. The interface used by the gctransformer is
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -379,6 +379,11 @@
self.total_gc_time = 0.0
self.gc_state = STATE_SCANNING
+
+ # if the GC is disabled, it runs only minor collections; major
+ # collections need to be manually triggered by explicitly calling
+ # collect()
+ self.enabled = True
#
# Two lists of all objects with finalizers. Actually they are lists
# of pairs (finalization_queue_nr, object). "probably young objects"
@@ -514,6 +519,15 @@
bigobj = self.nonlarge_max + 1
self.max_number_of_pinned_objects = self.nursery_size / (bigobj * 2)
+ def enable(self):
+ self.enabled = True
+
+ def disable(self):
+ self.enabled = False
+
+ def isenabled(self):
+ return self.enabled
+
def _nursery_memory_size(self):
extra = self.nonlarge_max + 1
return self.nursery_size + extra
@@ -750,22 +764,53 @@
"""Do a minor (gen=0), start a major (gen=1), or do a full
major (gen>=2) collection."""
if gen < 0:
- self._minor_collection() # dangerous! no major GC cycle progress
- elif gen <= 1:
- self.minor_collection_with_major_progress()
- if gen == 1 and self.gc_state == STATE_SCANNING:
+ # Dangerous! this makes no progress on the major GC cycle.
+ # If called too often, the memory usage will keep increasing,
+ # because we'll never completely fill the nursery (and so
+ # never run anything about the major collection).
+ self._minor_collection()
+ elif gen == 0:
+ # This runs a minor collection. This is basically what occurs
+ # when the nursery is full. If a major collection is in
+ # progress, it also runs one more step of it. It might also
+ # decide to start a major collection just now, depending on
+ # current memory pressure.
+ self.minor_collection_with_major_progress(force_enabled=True)
+ elif gen == 1:
+ # This is like gen == 0, but if no major collection is running,
+ # then it forces one to start now.
+ self.minor_collection_with_major_progress(force_enabled=True)
+ if self.gc_state == STATE_SCANNING:
self.major_collection_step()
else:
+ # This does a complete minor and major collection.
self.minor_and_major_collection()
self.rrc_invoke_callback()
+ def collect_step(self):
+ """
+ Do a single major collection step. Return True when the major collection
+ is completed.
- def minor_collection_with_major_progress(self, extrasize=0):
- """Do a minor collection. Then, if there is already a major GC
- in progress, run at least one major collection step. If there is
- no major GC but the threshold is reached, start a major GC.
+ This is meant to be used together with gc.disable(), to have a
+ fine-grained control on when the GC runs.
+ """
+ old_state = self.gc_state
+ self._minor_collection()
+ self.major_collection_step()
+ self.rrc_invoke_callback()
+ return rgc._encode_states(old_state, self.gc_state)
+
+ def minor_collection_with_major_progress(self, extrasize=0,
+ force_enabled=False):
+ """Do a minor collection. Then, if the GC is enabled and there
+ is already a major GC in progress, run at least one major collection
+ step. If there is no major GC but the threshold is reached, start a
+ major GC.
"""
self._minor_collection()
+ if not self.enabled and not force_enabled:
+ return
# If the gc_state is STATE_SCANNING, we're not in the middle
# of an incremental major collection. In that case, wait
@@ -2428,25 +2473,6 @@
# We also need to reset the GCFLAG_VISITED on prebuilt GC objects.
self.prebuilt_root_objects.foreach(self._reset_gcflag_visited, None)
#
- # Print statistics
- debug_start("gc-collect-done")
- debug_print("arenas: ",
- self.stat_ac_arenas_count, " => ",
- self.ac.arenas_count)
- debug_print("bytes used in arenas: ",
- self.ac.total_memory_used)
- debug_print("bytes raw-malloced: ",
- self.stat_rawmalloced_total_size, " => ",
- self.rawmalloced_total_size)
- debug_stop("gc-collect-done")
- self.hooks.fire_gc_collect(
- num_major_collects=self.num_major_collects,
- arenas_count_before=self.stat_ac_arenas_count,
- arenas_count_after=self.ac.arenas_count,
- arenas_bytes=self.ac.total_memory_used,
- rawmalloc_bytes_before=self.stat_rawmalloced_total_size,
- rawmalloc_bytes_after=self.rawmalloced_total_size)
- #
# Set the threshold for the next major collection to be when we
# have allocated 'major_collection_threshold' times more than
# we currently have -- but no more than 'max_delta' more than
@@ -2460,6 +2486,27 @@
total_memory_used + self.max_delta),
reserving_size)
#
+ # Print statistics
+ debug_start("gc-collect-done")
+ debug_print("arenas: ",
+ self.stat_ac_arenas_count, " => ",
+ self.ac.arenas_count)
+ debug_print("bytes used in arenas: ",
+ self.ac.total_memory_used)
+ debug_print("bytes raw-malloced: ",
+ self.stat_rawmalloced_total_size, " => ",
+ self.rawmalloced_total_size)
+ debug_print("next major collection threshold: ",
+ self.next_major_collection_threshold)
+ debug_stop("gc-collect-done")
+ self.hooks.fire_gc_collect(
+ num_major_collects=self.num_major_collects,
+ arenas_count_before=self.stat_ac_arenas_count,
+ arenas_count_after=self.ac.arenas_count,
+ arenas_bytes=self.ac.total_memory_used,
+ rawmalloc_bytes_before=self.stat_rawmalloced_total_size,
+ rawmalloc_bytes_after=self.rawmalloced_total_size)
+ #
# Max heap size: gives an upper bound on the threshold. If we
# already have at least this much allocated, raise MemoryError.
if bounded and self.threshold_reached(reserving_size):
diff --git a/rpython/memory/gc/test/test_direct.py \
b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -13,6 +13,7 @@
from rpython.memory.gc import minimark, incminimark
from rpython.memory.gctypelayout import zero_gc_pointers_inside, zero_gc_pointers
from rpython.rlib.debug import debug_print
+from rpython.rlib.test.test_debug import debuglog
import pdb
WORD = LONG_BIT // 8
@@ -770,4 +771,76 @@
assert elem.prev == lltype.nullptr(S)
assert elem.next == lltype.nullptr(S)
-
+ def test_collect_0(self, debuglog):
+ self.gc.collect(1) # start a major
+ debuglog.reset()
+ self.gc.collect(0) # do ONLY a minor
+ assert debuglog.summary() == {'gc-minor': 1}
+
+ def test_enable_disable(self, debuglog):
+ def large_malloc():
+ # malloc an object which is large enough to trigger a major collection
+ threshold = self.gc.next_major_collection_threshold
+ self.malloc(VAR, int(threshold/8))
+ summary = debuglog.summary()
+ debuglog.reset()
+ return summary
+ #
+ summary = large_malloc()
+ assert sorted(summary.keys()) == ['gc-collect-step', 'gc-minor']
+ #
+ self.gc.disable()
+ summary = large_malloc()
+ assert sorted(summary.keys()) == ['gc-minor']
+ #
+ self.gc.enable()
+ summary = large_malloc()
+ assert sorted(summary.keys()) == ['gc-collect-step', 'gc-minor']
+
+ def test_call_collect_when_disabled(self, debuglog):
+ # malloc an object and put it the old generation
+ s = self.malloc(S)
+ s.x = 42
+ self.stackroots.append(s)
+ self.gc.collect()
+ s = self.stackroots.pop()
+ #
+ self.gc.disable()
+ self.gc.collect(1) # start a major collect
+ assert sorted(debuglog.summary()) == ['gc-collect-step', 'gc-minor']
+ assert s.x == 42 # s is not freed yet
+ #
+ debuglog.reset()
+ self.gc.collect(1) # run one more step
+ assert sorted(debuglog.summary()) == ['gc-collect-step', 'gc-minor']
+ assert s.x == 42 # s is not freed yet
+ #
+ debuglog.reset()
+ self.gc.collect() # finish the major collection
+ summary = debuglog.summary()
+ assert sorted(debuglog.summary()) == ['gc-collect-step', 'gc-minor']
+ # s is freed
+ py.test.raises(RuntimeError, 's.x')
+
+ def test_collect_step(self, debuglog):
+ from rpython.rlib import rgc
+ n = 0
+ states = []
+ while True:
+ debuglog.reset()
+ val = self.gc.collect_step()
+ states.append((rgc.old_state(val), rgc.new_state(val)))
+ summary = debuglog.summary()
+ assert summary == {'gc-minor': 1, 'gc-collect-step': 1}
+ if rgc.is_done(val):
+ break
+ n += 1
+ if n == 100:
+ assert False, 'this looks like an endless loop'
+ #
+ assert states == [
+ (incminimark.STATE_SCANNING, incminimark.STATE_MARKING),
+ (incminimark.STATE_MARKING, incminimark.STATE_SWEEPING),
+ (incminimark.STATE_SWEEPING, incminimark.STATE_FINALIZING),
+ (incminimark.STATE_FINALIZING, incminimark.STATE_SCANNING)
+ ]
diff --git a/rpython/memory/gctransform/framework.py \
b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -309,6 +309,12 @@
self.collect_ptr = getfn(GCClass.collect.im_func,
[s_gc, annmodel.SomeInteger()], annmodel.s_None)
+ self.collect_step_ptr = getfn(GCClass.collect_step.im_func, [s_gc],
+ annmodel.SomeInteger())
+ self.enable_ptr = getfn(GCClass.enable.im_func, [s_gc], annmodel.s_None)
+ self.disable_ptr = getfn(GCClass.disable.im_func, [s_gc], annmodel.s_None)
+ self.isenabled_ptr = getfn(GCClass.isenabled.im_func, [s_gc],
+ annmodel.s_Bool)
self.can_move_ptr = getfn(GCClass.can_move.im_func,
[s_gc, SomeAddress()],
annmodel.SomeBool())
@@ -884,6 +890,28 @@
resultvar=op.result)
self.pop_roots(hop, livevars)
+ def gct_gc__collect_step(self, hop):
+ op = hop.spaceop
+ livevars = self.push_roots(hop)
+ hop.genop("direct_call", [self.collect_step_ptr, self.c_const_gc],
+ resultvar=op.result)
+ self.pop_roots(hop, livevars)
+
+ def gct_gc__enable(self, hop):
+ op = hop.spaceop
+ hop.genop("direct_call", [self.enable_ptr, self.c_const_gc],
+ resultvar=op.result)
+
+ def gct_gc__disable(self, hop):
+ op = hop.spaceop
+ hop.genop("direct_call", [self.disable_ptr, self.c_const_gc],
+ resultvar=op.result)
+
+ def gct_gc__isenabled(self, hop):
+ op = hop.spaceop
+ hop.genop("direct_call", [self.isenabled_ptr, self.c_const_gc],
+ resultvar=op.result)
+
def gct_gc_can_move(self, hop):
op = hop.spaceop
v_addr = hop.genop('cast_ptr_to_adr',
diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py
--- a/rpython/rlib/debug.py
+++ b/rpython/rlib/debug.py
@@ -1,5 +1,6 @@
import sys
import time
+from collections import Counter
from rpython.rlib.objectmodel import enforceargs
from rpython.rtyper.extregistry import ExtRegistryEntry
@@ -38,6 +39,23 @@
assert False, ("nesting error: no start corresponding to stop %r" %
(category,))
+ def reset(self):
+ # only for tests: empty the log
+ self[:] = []
+
+ def summary(self, flatten=False):
+ res = Counter()
+ def visit(lst):
+ for section, sublist in lst:
+ if section == 'debug_print':
+ continue
+ res[section] += 1
+ if flatten:
+ visit(sublist)
+ #
+ visit(self)
+ return res
+
def __repr__(self):
import pprint
return pprint.pformat(list(self))
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -13,6 +13,50 @@
# General GC features
collect = gc.collect
+enable = gc.enable
+disable = gc.disable
+isenabled = gc.isenabled
+
+def collect_step():
+ """
+ If the GC is incremental, run a single gc-collect-step.
+
+ Return an integer which encodes the starting and ending GC state. Use
+ rgc.{old_state,new_state,is_done} to decode it.
+
+ If the GC is not incremental, do a full collection and return a value on
+ which rgc.is_done() return True.
+ """
+ gc.collect()
+ return _encode_states(1, 0)
+
+def _encode_states(oldstate, newstate):
+ return oldstate << 8 | newstate
+
+def old_state(states):
+ return (states & 0xFF00) >> 8
+
+def new_state(states):
+ return states & 0xFF
+
+def is_done(states):
+ """
+ Return True if the return value of collect_step signals the end of a major
+ collection
+ """
+ old = old_state(states)
+ new = new_state(states)
+ return is_done__states(old, new)
+
+def is_done__states(oldstate, newstate):
+ "Like is_done, but takes oldstate and newstate explicitly"
+ # a collection is considered done when it ends up in the starting state
+ # (which is usually represented as 0). This logic works for incminimark,
+ # which is currently the only gc actually used and for which collect_step
+ # is implemented. In case we add more GC in the future, we might want to
+ # delegate this logic to the GC itself, but for now it is MUCH simpler to
+ # just write it in plain RPython.
+ return oldstate != 0 and newstate == 0
def set_max_heap_size(nbytes):
"""Limit the heap size to n bytes.
@@ -131,6 +175,44 @@
args_v = hop.inputargs(lltype.Signed)
return hop.genop('gc__collect', args_v, resulttype=hop.r_result)
+
+class EnableDisableEntry(ExtRegistryEntry):
+ _about_ = (gc.enable, gc.disable)
+
+ def compute_result_annotation(self):
+ from rpython.annotator import model as annmodel
+ return annmodel.s_None
+
+ def specialize_call(self, hop):
+ hop.exception_cannot_occur()
+ opname = self.instance.__name__
+ return hop.genop('gc__%s' % opname, hop.args_v, resulttype=hop.r_result)
+
+
+class IsEnabledEntry(ExtRegistryEntry):
+ _about_ = gc.isenabled
+
+ def compute_result_annotation(self):
+ from rpython.annotator import model as annmodel
+ return annmodel.s_Bool
+
+ def specialize_call(self, hop):
+ hop.exception_cannot_occur()
+ return hop.genop('gc__isenabled', hop.args_v, resulttype=hop.r_result)
+
+
+class CollectStepEntry(ExtRegistryEntry):
+ _about_ = collect_step
+
+ def compute_result_annotation(self):
+ from rpython.annotator import model as annmodel
+ return annmodel.SomeInteger()
+
+ def specialize_call(self, hop):
+ hop.exception_cannot_occur()
+ return hop.genop('gc__collect_step', hop.args_v, resulttype=hop.r_result)
+
+
class SetMaxHeapSizeEntry(ExtRegistryEntry):
_about_ = set_max_heap_size
diff --git a/rpython/rlib/test/test_debug.py b/rpython/rlib/test/test_debug.py
--- a/rpython/rlib/test/test_debug.py
+++ b/rpython/rlib/test/test_debug.py
@@ -1,5 +1,5 @@
-
import py
+import pytest
from rpython.rlib.debug import (check_annotation, make_sure_not_resized,
debug_print, debug_start, debug_stop,
have_debug_prints, debug_offset, debug_flush,
@@ -10,6 +10,12 @@
from rpython.rlib import debug
from rpython.rtyper.test.test_llinterp import interpret, gengraph
+@pytest.fixture
+def debuglog(monkeypatch):
+ dlog = debug.DebugLog()
+ monkeypatch.setattr(debug, '_log', dlog)
+ return dlog
+
def test_check_annotation():
class Error(Exception):
pass
@@ -94,7 +100,7 @@
py.test.raises(NotAListOfChars, "interpret(g, [3])")
-def test_debug_print_start_stop():
+def test_debug_print_start_stop(debuglog):
def f(x):
debug_start("mycat")
debug_print("foo", 2, "bar", x)
@@ -103,22 +109,27 @@
debug_offset() # should not explode at least
return have_debug_prints()
- try:
- debug._log = dlog = debug.DebugLog()
- res = f(3)
- assert res is True
- finally:
- debug._log = None
- assert dlog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])]
+ res = f(3)
+ assert res is True
+ assert debuglog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])]
+ debuglog.reset()
- try:
- debug._log = dlog = debug.DebugLog()
- res = interpret(f, [3])
- assert res is True
- finally:
- debug._log = None
- assert dlog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])]
+ res = interpret(f, [3])
+ assert res is True
+ assert debuglog == [("mycat", [('debug_print', 'foo', 2, 'bar', 3)])]
+def test_debuglog_summary(debuglog):
+ debug_start('foo')
+ debug_start('bar') # this is nested, so not counted in the summary by default
+ debug_stop('bar')
+ debug_stop('foo')
+ debug_start('foo')
+ debug_stop('foo')
+ debug_start('bar')
+ debug_stop('bar')
+ #
+ assert debuglog.summary() == {'foo': 2, 'bar': 1}
+ assert debuglog.summary(flatten=True) == {'foo': 2, 'bar': 2}
def test_debug_start_stop_timestamp():
import time
diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py
--- a/rpython/rlib/test/test_rgc.py
+++ b/rpython/rlib/test/test_rgc.py
@@ -39,6 +39,45 @@
assert res is None
+def test_enable_disable():
+ def f():
+ gc.enable()
+ a = gc.isenabled()
+ gc.disable()
+ b = gc.isenabled()
+ return a and not b
+
+ t, typer, graph = gengraph(f, [])
+ blockops = list(graph.iterblockops())
+ opnames = [op.opname for block, op in blockops
+ if op.opname.startswith('gc__')]
+ assert opnames == ['gc__enable', 'gc__isenabled',
+ 'gc__disable', 'gc__isenabled']
+ res = interpret(f, [])
+ assert res
+
+def test_collect_step():
+ def f():
+ return rgc.collect_step()
+
+ assert f()
+ t, typer, graph = gengraph(f, [])
+ blockops = list(graph.iterblockops())
+ opnames = [op.opname for block, op in blockops
+ if op.opname.startswith('gc__')]
+ assert opnames == ['gc__collect_step']
+ res = interpret(f, [])
+ assert res
+
+def test__encode_states():
+ val = rgc._encode_states(42, 43)
+ assert rgc.old_state(val) == 42
+ assert rgc.new_state(val) == 43
+ assert not rgc.is_done(val)
+ #
+ val = rgc.collect_step()
+ assert rgc.is_done(val)
+
def test_can_move():
T0 = lltype.GcStruct('T')
T1 = lltype.GcArray(lltype.Float)
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -819,6 +819,18 @@
def op_gc__collect(self, *gen):
self.heap.collect(*gen)
+ def op_gc__collect_step(self):
+ return self.heap.collect_step()
+
+ def op_gc__enable(self):
+ self.heap.enable()
+
+ def op_gc__disable(self):
+ self.heap.disable()
+
+ def op_gc__isenabled(self):
+ return self.heap.isenabled()
+
def op_gc_heap_stats(self):
raise NotImplementedError
diff --git a/rpython/rtyper/lltypesystem/llheap.py \
b/rpython/rtyper/lltypesystem/llheap.py
--- a/rpython/rtyper/lltypesystem/llheap.py
+++ b/rpython/rtyper/lltypesystem/llheap.py
@@ -5,7 +5,7 @@
setfield = setattr
from operator import setitem as setarrayitem
-from rpython.rlib.rgc import can_move, collect, add_memory_pressure
+from rpython.rlib.rgc import can_move, collect, enable, disable, isenabled, \
add_memory_pressure, collect_step
def setinterior(toplevelcontainer, inneraddr, INNERTYPE, newvalue,
offsets=None):
diff --git a/rpython/rtyper/lltypesystem/lloperation.py \
b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -456,6 +456,10 @@
# __________ GC operations __________
'gc__collect': LLOp(canmallocgc=True),
+ 'gc__collect_step': LLOp(canmallocgc=True),
+ 'gc__enable': LLOp(),
+ 'gc__disable': LLOp(),
+ 'gc__isenabled': LLOp(),
'gc_free': LLOp(),
'gc_fetch_exception': LLOp(),
'gc_restore_exception': LLOp(),
diff --git a/rpython/translator/c/test/test_newgc.py \
b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1812,6 +1812,92 @@
res = self.run("ignore_finalizer")
assert res == 1 # translated: x1 is removed from the list
+ def define_enable_disable(self):
+ class Counter(object):
+ val = 0
+ counter = Counter()
+ class X(object):
+ def __del__(self):
+ counter.val += 1
+ def f(should_disable):
+ x1 = X()
+ rgc.collect() # make x1 old
+ assert not rgc.can_move(x1)
+ x1 = None
+ #
+ if should_disable:
+ gc.disable()
+ assert not gc.isenabled()
+ # try to trigger a major collection
+ N = 100 # this should be enough, increase if not
+ lst = []
+ for i in range(N):
+ lst.append(chr(i%256) * (1024*1024))
+ #print i, counter.val
+ #
+ gc.enable()
+ assert gc.isenabled()
+ return counter.val
+ return f
+
+ def test_enable_disable(self):
+ # first, run with normal gc. If the assert fails it means that in the
+ # loop we don't allocate enough mem to trigger a major collection. Try
+ # to increase N
+ deleted = self.run("enable_disable", 0)
+ assert deleted == 1, 'This should not fail, try to increment N'
+ #
+ # now, run with gc.disable: this should NOT free x1
+ deleted = self.run("enable_disable", 1)
+ assert deleted == 0
+
+ def define_collect_step(self):
+ class Counter(object):
+ val = 0
+ counter = Counter()
+ class X(object):
+ def __del__(self):
+ counter.val += 1
+ def f():
+ x1 = X()
+ rgc.collect() # make x1 old
+ assert not rgc.can_move(x1)
+ x1 = None
+ #
+ gc.disable()
+ n = 0
+ states = []
+ while True:
+ n += 1
+ val = rgc.collect_step()
+ states.append((rgc.old_state(val), rgc.new_state(val)))
+ if rgc.is_done(val):
+ break
+ if n == 100:
+ print 'Endless loop!'
+ assert False, 'this looks like an endless loop'
+
+ if n < 4: # we expect at least 4 steps
+ print 'Too few steps! n =', n
+ assert False
+
+ # check that the state transitions are reasonable
+ first_state, _ = states[0]
+ for i, (old_state, new_state) in enumerate(states):
+ is_last = (i == len(states) - 1)
+ is_valid = False
+ if is_last:
+ assert old_state != new_state == first_state
+ else:
+ assert new_state == old_state or new_state == old_state+1
+
+ return counter.val
+ return f
+
+ def test_collect_step(self):
+ deleted = self.run("collect_step")
+ assert deleted == 1
+
def define_total_gc_time(cls):
def f():
l = []
diff --git a/rpython/translator/goal/gcbench.py b/rpython/translator/goal/gcbench.py
--- a/rpython/translator/goal/gcbench.py
+++ b/rpython/translator/goal/gcbench.py
@@ -44,8 +44,9 @@
# - Results are sensitive to locking cost, but we dont
# check for proper locking
import time
+import gc
-USAGE = """gcbench [num_repetitions] [--depths=N,N,N..] [--threads=N]"""
+USAGE = """gcbench [num_repetitions] [--depths=N,N,N..] [--threads=N] \
[--gc=off|--gc=manual]""" ENABLE_THREADS = True
@@ -173,6 +174,7 @@
depths = DEFAULT_DEPTHS
threads = 0
repeatcount = 1
+ gc_policy = 'on'
for arg in argv[1:]:
if arg.startswith('--threads='):
arg = arg[len('--threads='):]
@@ -189,13 +191,22 @@
depths = [int(s) for s in arg]
except ValueError:
return argerror()
+ elif arg.startswith('--gc=off'):
+ gc_policy = 'off'
+ elif arg.startswith('--gc=manual'):
+ gc_policy = 'manual'
else:
try:
repeatcount = int(arg)
except ValueError:
return argerror()
+ #
+ if gc_policy == 'off' or gc_policy == 'manual':
+ gc.disable()
for i in range(repeatcount):
main(depths, threads)
+ if gc_policy == 'manual':
+ gc.collect(1)
return 0
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic