[prev in list] [next in list] [prev in thread] [next in thread]
List: sptk-commits
Subject: r1628 - in trunk: examples sptk4 src/utils
From: alexey () mail ! total-knowledge ! com
Date: 2011-09-20 3:28:59
Message-ID: courier.000000004E78087B.00001536 () mail ! total-knowledge ! com
[Download RAW message or body]
Author: alexey
Date: 2011-09-19 20:28:59 -0700 (Mon, 19 Sep 2011)
New Revision: 1628
Added:
trunk/examples/memory_map_records.cpp
Modified:
trunk/examples/CMakeLists.txt
trunk/sptk4/CMemoryMappedFile.h
trunk/sptk4/CMemoryMappedRecords.h
trunk/src/utils/CMemoryMappedFile.cpp
trunk/src/utils/CMemoryMappedRecords.cpp
Log:
Debugging CMemoryMappedRecords
Modified: trunk/examples/CMakeLists.txt
===================================================================
--- trunk/examples/CMakeLists.txt 2011-09-19 05:18:52 UTC (rev 1627)
+++ trunk/examples/CMakeLists.txt 2011-09-20 03:28:59 UTC (rev 1628)
@@ -21,6 +21,7 @@
logfile_test
memory_map_buffer
memory_map_buffer_performance
+ memory_map_records
safe_read_buffer
snappy_test
syslog_test
Added: trunk/examples/memory_map_records.cpp
===================================================================
--- trunk/examples/memory_map_records.cpp (rev 0)
+++ trunk/examples/memory_map_records.cpp 2011-09-20 03:28:59 UTC (rev 1628)
@@ -0,0 +1,113 @@
+/***************************************************************************
+ SIMPLY POWERFUL TOOLKIT (SPTK)
+ memory_map_records.cpp - description
+ -------------------
+ begin : January 3, 2003
+ copyright : (C) 2003-2011 by Alexey Parshin
+ email : alexeyp@gmail.com
+ ***************************************************************************/
+
+/***************************************************************************
+ This library is free software; you can redistribute it and/or modify it
+ under the terms of the GNU Library General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or (at
+ your option) any later version.
+
+ This library is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library
+ General Public License for more details.
+
+ You should have received a copy of the GNU Library General Public License
+ along with this library; if not, write to the Free Software Foundation,
+ Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
+
+ Please report all bugs and problems to "alexeyp@gmail.com"
+ ***************************************************************************/
+
+#include <stdio.h>
+#include <sptk4/cutils>
+#include <iostream>
+#include <sptk4/CMemoryMappedRecords.h>
+
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include <fcntl.h>
+
+using namespace std;
+using namespace sptk;
+
+struct TestRecord
+{
+ uint64_t id;
+ char text[128];
+ uint32_t data[256];
+};
+
+int main(int, const char**)
+{
+ string directory("/tmp/memory_map_records.test");
+ mkdir(directory.c_str(), 0777);
+
+ /// Define records object that works with chunks 128 records each.
+ /// Number of records in object is unlimited.
+ CMemoryMappedRecords records(directory, 16384, sizeof(TestRecord));
+
+ try {
+ records.open();
+
+ uint32_t recordCount = records.recordCount();
+ uint32_t totalRecords = 1024 * 256;
+
+ if (!recordCount) {
+ /// No records, creating them
+ uint64_t startedMS = uint64_t(CDateTime::NowCTimeMS() * 1000);
+ for (unsigned i = 0; i < totalRecords; i++) {
+ TestRecord* record = (TestRecord*) records.allocate();
+ record->id = i;
+ sprintf(record->text, "Record %i", i);
+ }
+ uint64_t endedMS = uint64_t(CDateTime::NowCTimeMS() * 1000);
+ cout << "MMR: " << totalRecords << " allocated for " << (endedMS - \
startedMS) << " ms" << endl; +
+ sleep(10);
+
+ /// Test for regular mallocs
+ startedMS = uint64_t(CDateTime::NowCTimeMS() * 1000);
+ for (unsigned i = 0; i < totalRecords; i++) {
+ TestRecord* record = (TestRecord*) malloc(sizeof(TestRecord));
+ record->id = i;
+ sprintf(record->text, "Record %i", i);
+ }
+ endedMS = uint64_t(CDateTime::NowCTimeMS() * 1000);
+ cout << "malloc: " << totalRecords << " allocated for " << (endedMS - \
startedMS) << " ms" << endl; + }
+
+ recordCount = records.recordCount();
+
+ /// Get the list of allocated records
+ vector<void*> allocatedRecords;
+ records.getRecords(allocatedRecords);
+
+ /// Print several records
+ uint32_t printCount = recordCount > 10 ? 10 : recordCount;
+ for (unsigned i = 0; i < printCount; i++) {
+ TestRecord* record = (TestRecord*) allocatedRecords[i];
+ cout << record->id << " : " << record->text << endl;
+ }
+
+ /// Deallocate every the record (also can be done much faster with \
deallocateAll()) + for (unsigned i = 0; i < recordCount; i++)
+ records.deallocate(allocatedRecords[i]);
+
+ /// Remove empty record chunks
+ records.compact();
+
+ records.close();
+ } catch (exception& e) {
+ cerr << "ERROR: " << e.what() << endl;
+ }
+
+ return 0;
+}
Modified: trunk/sptk4/CMemoryMappedFile.h
===================================================================
--- trunk/sptk4/CMemoryMappedFile.h 2011-09-19 05:18:52 UTC (rev 1627)
+++ trunk/sptk4/CMemoryMappedFile.h 2011-09-20 03:28:59 UTC (rev 1628)
@@ -96,7 +96,8 @@
/// @brief Closes memory-mapped file
///
/// Releases memory mapping and closes memory-mapped file
- virtual void close();
+ /// @param unlinkFile bool, If true then file is deleted after close memory \
mapping + virtual void close(bool unlinkFile=false);
/// @brief Returns true if memory-mapped file is opened
bool active() const;
Modified: trunk/sptk4/CMemoryMappedRecords.h
===================================================================
--- trunk/sptk4/CMemoryMappedRecords.h 2011-09-19 05:18:52 UTC (rev 1627)
+++ trunk/sptk4/CMemoryMappedRecords.h 2011-09-20 03:28:59 UTC (rev 1628)
@@ -59,12 +59,12 @@
{
CCriticalSection m_lock;
std::string m_directory;
- std::map<uint32_t, CMemoryMappedChunk*> m_allChunks;
- std::map<uint32_t, CMemoryMappedChunk*> m_incompleteChunks;
+ std::map<uint64_t, CMemoryMappedChunk*> m_allChunks;
+ std::map<uint64_t, CMemoryMappedChunk*> m_incompleteChunks;
uint32_t m_recordsPerChunk;
uint32_t m_recordSize;
bool m_active;
- uint32_t m_nextChunkId;
+ uint32_t m_nextChunkIndex;
public:
/// @brief Constructor
@@ -102,13 +102,26 @@
return m_directory;
}
- /// @brief Allocates new record, and returns record ptr and record id
- /// @param id uint64_t&, Unique record id
- void* newRecord(uint64_t& id);
+ /// @brief Allocates new record, and returns pointer to allocated memory
+ void* allocate();
/// @brief Deallocates record memory
- /// @param id uint64_t, Unique record id
- void deleteRecord(uint64_t id);
+ /// @param record void*, record pointer
+ void deallocate(void* record);
+
+ /// @brief Deallocates all records memory
+ /// @param record void*, record pointer
+ void deallocateAll();
+
+ /// @brief Returns total number of allocated records
+ uint32_t recordCount();
+
+ /// @brief Returns all allocated records
+ /// @param allocatedRecords std::vector<void*>&, Output vector of all allocated \
records + void getRecords(std::vector<void*>& allocatedRecords);
+
+ /// @brief Removes empty chunks that don't contain any records
+ void compact();
};
/// @}
Modified: trunk/src/utils/CMemoryMappedFile.cpp
===================================================================
--- trunk/src/utils/CMemoryMappedFile.cpp 2011-09-19 05:18:52 UTC (rev 1627)
+++ trunk/src/utils/CMemoryMappedFile.cpp 2011-09-20 03:28:59 UTC (rev 1628)
@@ -180,7 +180,7 @@
#endif
}
-void CMemoryMappedFile::close()
+void CMemoryMappedFile::close(bool unlinkFile)
{
#ifndef WIN32
if (m_buffer)
@@ -195,6 +195,9 @@
CloseHandle(m_file);
#endif
+ if (unlinkFile)
+ unlink(m_fileName.c_str());
+
m_buffer = NULL;
m_file = INVALID_HANDLE_VALUE;
m_fileName = "";
Modified: trunk/src/utils/CMemoryMappedRecords.cpp
===================================================================
--- trunk/src/utils/CMemoryMappedRecords.cpp 2011-09-19 05:18:52 UTC (rev 1627)
+++ trunk/src/utils/CMemoryMappedRecords.cpp 2011-09-20 03:28:59 UTC (rev 1628)
@@ -50,37 +50,71 @@
WAS_FULL
};
protected:
- CCriticalSection m_lock;
+
std::string m_fileName;
uint32_t m_maxRecords;
uint32_t m_recordSize;
std::list<uint32_t> m_available;
+ uint8_t* m_allocated;
uint8_t* m_records;
public:
CMemoryMappedChunk(std::string fileName, uint32_t maxRecords=0, uint32_t \
recordSize=0); void open() throw (std::exception);
void create() throw (std::exception);
- void* allocate(uint32_t& recno);
- CHUNK_STATE deallocate(uint32_t recno);
- bool full();
- bool empty();
+ void* allocate();
+ void deallocate(void* record);
+
+ void deallocateAll()
+ {
+ memset(m_allocated, 'F', m_maxRecords);
+ m_available.clear();
+ for (uint32_t i = 0; i < m_maxRecords; i++)
+ m_available.push_back(i);
+ }
+
+ uint32_t recordCount() const
+ {
+ return m_maxRecords - m_available.size();
+ }
+
+ bool full() const
+ {
+ return m_available.empty();
+ }
+
+ bool empty() const
+ {
+ return m_available.size() == m_maxRecords;
+ }
+
+ void getRecords(vector<void*>& allocatedRecords);
+
+ uint64_t id()
+ {
+ return uint64_t(m_records);
+ }
};
}
CMemoryMappedChunk::CMemoryMappedChunk(std::string fileName, uint32_t maxRecords, \
uint32_t recordSize) :
m_fileName(fileName), m_maxRecords(maxRecords), m_recordSize(recordSize)
{
- for (uint32_t i = 0; i < m_maxRecords; i++)
- m_available.push_back(i);
}
void CMemoryMappedChunk::create() throw (std::exception)
{
- CMemoryMappedFile::open(m_fileName, m_recordSize * m_maxRecords + 4);
+ /// File contains:
+ /// uint32_t max records
+ /// uint32_t record size
+ /// uint8_t[max records] allocated record flags
+ /// uint8_t[max records*record_size] records
+ CMemoryMappedFile::open(m_fileName, 8 + m_maxRecords + m_recordSize * \
m_maxRecords); uint32_t* header = (uint32_t *) data();
header[0] = m_maxRecords;
header[1] = m_recordSize;
- m_records = (uint8_t*) (header + 2);
+ m_allocated = (uint8_t*) (header + 2);
+ m_records = m_allocated + m_maxRecords;
+ deallocateAll();
}
void CMemoryMappedChunk::open() throw (std::exception)
@@ -89,48 +123,49 @@
uint32_t* header = (uint32_t *) data();
m_maxRecords = header[0];
m_recordSize = header[1];
- m_records = (uint8_t*) (header + 2);
+ m_allocated = (uint8_t*) (header + 2);
+ m_records = m_allocated + m_maxRecords;
+ m_available.clear();
+ for (uint32_t i = 0; i < m_maxRecords; i++) {
+ if (m_allocated[i] == 'F')
+ m_available.push_back(i);
+ }
}
-void* CMemoryMappedChunk::allocate(uint32_t& recno)
+void* CMemoryMappedChunk::allocate()
{
- CODE_GUARD(m_lock);
if (m_available.empty())
return NULL; /// Chunk is full
- recno = m_available.front();
+ uint32_t recno = m_available.front();
m_available.pop_front();
- return (uint8_t *) data() + recno * m_recordSize;
+ m_allocated[recno] = 'A';
+ return m_records + recno * m_recordSize;
}
-CMemoryMappedChunk::CHUNK_STATE CMemoryMappedChunk::deallocate(uint32_t recno)
+void CMemoryMappedChunk::deallocate(void* record)
{
- CODE_GUARD(m_lock);
- CHUNK_STATE rc = WAS_INCOMPLETE;
- if (m_available.empty())
- rc = WAS_FULL;
- if (recno < m_maxRecords)
- m_available.push_back(recno);
- return rc;
+ uint32_t recno = ((uint8_t*)record - m_records) / m_recordSize;
+ if (recno >= m_maxRecords)
+ EXCEPTION("Memory doesn't belong to the memory manager");
+ m_available.push_back(recno);
+ m_allocated[recno] = 'F';
}
-bool CMemoryMappedChunk::full()
+void CMemoryMappedChunk::getRecords(vector<void*>& allocatedRecords)
{
- CODE_GUARD(m_lock);
- return m_available.empty();
+ uint8_t* ptr = m_records;
+ for (uint32_t i = 0; i < m_maxRecords; i++, ptr += m_recordSize) {
+ if (m_allocated[i] == 'A')
+ allocatedRecords.push_back(ptr);
+ }
}
-bool CMemoryMappedChunk::empty()
-{
- CODE_GUARD(m_lock);
- return m_available.size() == m_maxRecords;
-}
-
CMemoryMappedRecords::CMemoryMappedRecords(std::string directory, uint32_t \
recordsPerChunk, uint32_t recordSize) : m_directory(directory),
m_recordsPerChunk(recordsPerChunk),
m_recordSize(recordSize),
m_active(false),
- m_nextChunkId(1)
+ m_nextChunkIndex(1)
{
}
@@ -141,6 +176,12 @@
void CMemoryMappedRecords::open() throw (std::exception)
{
+ {
+ CODE_GUARD(m_lock);
+ if (m_active)
+ throw CException("Records object is already active");
+ }
+
CDirectoryDS chunksDS;
chunksDS.directory(m_directory);
chunksDS.showPolicy(DDS_HIDE_DOT_FILES|DDS_HIDE_DIRECTORIES);
@@ -150,21 +191,19 @@
while (!chunksDS.eof()) {
try {
string name = chunksDS["Name"].asString();
- string fileName = m_directory + "/" + name;
- uint64_t id = string2int(name);
- if (id == 0) {
- chunksDS.next();
- continue;
+ uint32_t index = string2int(name);
+ if (index) {
+ string fileName = m_directory + "/" + name;
+ CODE_GUARD(m_lock);
+ CMemoryMappedChunk* chunk = new CMemoryMappedChunk(fileName, \
m_recordsPerChunk, m_recordSize); + chunk->open();
+ uint64_t chunkId = chunk->id();
+ m_allChunks[chunkId] = chunk;
+ if (!chunk->full())
+ m_incompleteChunks[chunkId] = chunk;
+ if (m_nextChunkIndex <= index)
+ m_nextChunkIndex = index + 1;
}
-
- CODE_GUARD(m_lock);
- CMemoryMappedChunk* chunk = new CMemoryMappedChunk(fileName, \
m_recordsPerChunk, m_recordSize);
- chunk->open();
- m_allChunks[id] = chunk;
- if (!chunk->full())
- m_incompleteChunks[id] = chunk;
- if (m_nextChunkId <= id)
- m_nextChunkId = id + 1;
}
catch (exception& e) {
cerr << e.what() << endl;
@@ -178,81 +217,139 @@
void CMemoryMappedRecords::sync(bool async) throw (std::exception)
{
- std::map<uint32_t, CMemoryMappedChunk*>::iterator itor = m_allChunks.begin();
- for (; itor != m_allChunks.end(); itor++) {
- CMemoryMappedChunk* chunk = itor->second;
- chunk->sync(async);
+ if (m_active) {
+ std::map<uint64_t, CMemoryMappedChunk*>::iterator itor = \
m_allChunks.begin(); + for (; itor != m_allChunks.end(); itor++) {
+ CMemoryMappedChunk* chunk = itor->second;
+ chunk->sync(async);
+ }
}
}
void CMemoryMappedRecords::close()
{
CODE_GUARD(m_lock);
- std::map<uint32_t, CMemoryMappedChunk*>::iterator itor = m_allChunks.begin();
- for (; itor != m_allChunks.end(); itor++) {
- CMemoryMappedChunk* chunk = itor->second;
- chunk->close();
- delete chunk;
+
+ if (m_active) {
+ std::map<uint64_t, CMemoryMappedChunk*>::iterator itor = \
m_allChunks.begin(); + for (; itor != m_allChunks.end(); itor++) {
+ CMemoryMappedChunk* chunk = itor->second;
+ chunk->close();
+ delete chunk;
+ }
+ m_allChunks.clear();
+ m_incompleteChunks.clear();
+ m_active = false;
}
- m_allChunks.clear();
- m_incompleteChunks.clear();
- m_active = false;
}
-void* CMemoryMappedRecords::newRecord(uint64_t& id)
+void* CMemoryMappedRecords::allocate()
{
- union {
- uint32_t idparts[2];
- uint64_t full;
- } rec_id;
+ CODE_GUARD(m_lock);
+ if (!m_active)
+ throw CException("Records object is not active");
- CODE_GUARD(m_lock);
CMemoryMappedChunk* chunk(NULL);
while (!m_incompleteChunks.empty()) {
- map<uint32_t, CMemoryMappedChunk*>::iterator itor = \
m_incompleteChunks.begin(); + map<uint64_t, CMemoryMappedChunk*>::iterator \
itor = m_incompleteChunks.begin(); CMemoryMappedChunk* chunk = itor->second;
- void* record = chunk->allocate(rec_id.idparts[1]);
- if (record) {
- rec_id.idparts[0] = itor->first;
- id = rec_id.full;
+ void* record = chunk->allocate();
+ if (record)
return record;
- }
/// Chunk is full
m_incompleteChunks.erase(itor->first);
}
/// No incomplete chunks, add one
- uint64_t chunkId = m_nextChunkId;
stringstream fileName;
- fileName << m_directory << "/" << chunkId;
+ fileName << m_directory << "/" << m_nextChunkIndex;
chunk = new CMemoryMappedChunk(fileName.str(), m_recordsPerChunk, m_recordSize);
- chunk->open();
- m_allChunks[chunkId] = chunk;
- m_incompleteChunks[chunkId] = chunk;
- m_nextChunkId++;
+ chunk->create();
+ m_allChunks[chunk->id()] = chunk;
+ m_incompleteChunks[chunk->id()] = chunk;
+ m_nextChunkIndex++;
- void* record = chunk->allocate(rec_id.idparts[1]);
- rec_id.idparts[0] = chunkId;
+ return chunk->allocate();
+}
- id = rec_id.full;
- return record;
+void CMemoryMappedRecords::deallocate(void* record)
+{
+ uint64_t chunkId = uint64_t(record);
+
+ CODE_GUARD(m_lock);
+ if (!m_active)
+ throw CException("Records object is not active");
+
+ map<uint64_t, CMemoryMappedChunk*>::iterator itor = \
m_allChunks.upper_bound(chunkId); + itor--;
+ CMemoryMappedChunk* chunk = itor->second;
+ if (chunk->full()) {
+ /// Chunk was full, but now it has available record
+ m_incompleteChunks[chunk->id()] = chunk;
+ }
+ chunk->deallocate(record);
}
-void CMemoryMappedRecords::deleteRecord(uint64_t id)
+uint32_t CMemoryMappedRecords::recordCount()
{
- union {
- uint32_t idparts[2];
- uint64_t full;
- } rec_id;
+ uint32_t count = 0;
- rec_id.full = id;
- uint32_t chunkId = rec_id.idparts[0];
+ if (!m_active)
+ throw CException("Records object is not active");
- map<uint32_t, CMemoryMappedChunk*>::iterator itor = m_allChunks.find(chunkId);
- if (itor == m_allChunks.end())
- throw CException("Invalid chunk id");
+ CODE_GUARD(m_lock);
+ std::map<uint64_t, CMemoryMappedChunk*>::iterator itor = m_allChunks.begin();
+ for (; itor != m_allChunks.end(); itor++) {
+ CMemoryMappedChunk* chunk = itor->second;
+ count += chunk->recordCount();
+ }
- CMemoryMappedChunk* chunk = itor->second;
- chunk->deallocate(rec_id.idparts[1]);
-}
\ No newline at end of file
+ return count;
+}
+
+void CMemoryMappedRecords::getRecords(vector<void*>& allocatedRecords)
+{
+ allocatedRecords.clear();
+
+ CODE_GUARD(m_lock);
+ if (!m_active)
+ throw CException("Records object is not active");
+
+ std::map<uint64_t, CMemoryMappedChunk*>::iterator itor = m_allChunks.begin();
+ for (; itor != m_allChunks.end(); itor++) {
+ CMemoryMappedChunk* chunk = itor->second;
+ chunk->getRecords(allocatedRecords);
+ }
+}
+
+void CMemoryMappedRecords::deallocateAll()
+{
+ CODE_GUARD(m_lock);
+ if (!m_active)
+ throw CException("Records object is not active");
+
+ std::map<uint64_t, CMemoryMappedChunk*>::iterator itor = m_allChunks.begin();
+ for (; itor != m_allChunks.end(); itor++) {
+ CMemoryMappedChunk* chunk = itor->second;
+ if (!chunk->empty())
+ chunk->deallocateAll();
+ }
+}
+
+void CMemoryMappedRecords::compact()
+{
+ CODE_GUARD(m_lock);
+ if (!m_active)
+ throw CException("Records object is not active");
+
+ std::map<uint64_t, CMemoryMappedChunk*>::iterator itor = m_allChunks.begin();
+ for (; itor != m_allChunks.end(); itor++) {
+ CMemoryMappedChunk* chunk = itor->second;
+ if (chunk->empty()) {
+ chunk->close(true);
+ m_allChunks.erase(chunk->id());
+ m_incompleteChunks.erase(chunk->id());
+ }
+ }
+}
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic