Click here to Skip to main content
Click here to Skip to main content
 
Add your own
alternative version

Tagged as

Go to top

Building an embedded database engine in C#

, 10 Jun 2009
DbfDotNet is a very fast and compact fully managed standalone database/entity framework, for the .Net Framework.
using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using System.Diagnostics;

namespace DbfDotNet.Core
{
    internal abstract class ClusteredFile
    {
        protected UInt32 mRecordCount;
        internal UInt32 mCurrentRecordNoPosition;
        private Dictionary<UInt32, LinkedListNode<RecordHolder>> mRecordsByRecordNo;
        private LinkedList<RecordHolder> mRecordsByLastUsed;
        private int mConstructorThreadId;
        private bool mIsDisposed = false;    // To detect redundant calls
        internal int mRecordWidth = 0;
        internal int mHeaderWidth = 0;
        private Stream mStream;
        protected FileMode mFileMode;
        protected FileAccess mFileAccess;
        protected FileShare mFileShare;
        internal List<ColumnDefinition> mColumns;       
        protected static Dictionary<string, ClusteredFile> mOpenFiles;
        private int mReaderCount;
        internal string mOriginalFile;

        const double PERCENT_CACHED_RECORDS = 2.5 / 100; // 2.5% normally
        const int MIN_CACHED_RECORDS = 100;
        const int MAX_CACHED_RECORDS = 10000; 
        
        static ClusteredFile()
        {
            // the filesystem ignore case
            mOpenFiles = new Dictionary<string, ClusteredFile>(StringComparer.OrdinalIgnoreCase);
        }

        protected ClusteredFile()
        {
            mConstructorThreadId = System.Threading.Thread.CurrentThread.ManagedThreadId;
        }
        
        private void Seek(UInt32 newRecordNo)
        {
            Debug.Assert(newRecordNo<UInt32.MaxValue,"Invalid RecordNo");
            if (newRecordNo == mCurrentRecordNoPosition) return;
            mStream.Seek(newRecordNo * mRecordWidth + mHeaderWidth, SeekOrigin.Begin);
            mCurrentRecordNoPosition = newRecordNo;
        }
        
        #region IDisposable Members

        protected virtual void OnDisposing()
        {
            Flush();
        }

        private void Dispose(bool disposing)
        {
            if (mIsDisposed) return;
            if (disposing)
            {
                OnDisposing();
                mStream.Close();
                GC.SuppressFinalize(this);
            }
            mIsDisposed = true;
            mStream = null;
        }

        public virtual void Flush()
        {
            if (mFileAccess != FileAccess.Read)
            {
                var values = mRecordsByRecordNo.Values;

                foreach (var r in values)
                {
                    Flush(r);
                }
                WriteHeader();
            }
        }

        private void Flush(LinkedListNode<RecordHolder> node)
        {
            var recordHolder = node.Value;
            var recordNo = recordHolder.RecordNo;
            var record = recordHolder.Record;
            if (record != null)
                InternalFillWriteBuffer(recordHolder, record);
            if (recordHolder.IsModified())
                recordHolder.Save();
        }



        ~ClusteredFile()
        {
            Dispose(false);
        }
        
        #endregion

        internal virtual protected Record InternalGetRecord(UInt32 recordNo, bool returnNullIfNotInCache)
        {
            Debug.Assert(recordNo < UInt32.MaxValue, "Invalid RecordNo");
            LinkedListNode<RecordHolder> linkListNode = null;
            RecordHolder holder = null;
            Record record = null;

            if (mRecordsByRecordNo.TryGetValue(recordNo, out linkListNode))
            {
                holder = linkListNode.Value;
                record = holder.Record;
                if (record != null)
                {
                    SetLastUsed(linkListNode);
                    return record;
                }
                else
                {
                    // we're going to add a new one in one sec
                    RemoveLastUsed(linkListNode);
                }
            }

            if (returnNullIfNotInCache) return null;
            // we're here because the record was disposed or never created

            var newRecord = OnCreateNewRecord(/*isnew*/false, recordNo);


            if (holder == null)
            {
                holder = new RecordHolder(this, recordNo, newRecord);
                linkListNode = new LinkedListNode<RecordHolder>(holder);
                SetLastUsed(linkListNode);
                mRecordsByRecordNo[recordNo] = linkListNode;
#if DUMP_FINALIZE
                    System.Diagnostics.Trace.WriteLine(this.mOriginalFile + " Save Record #" + recordNo);
#endif
            }
            else
            {
                // holder exist but object doesn't we expect that a finalized happened
                if (holder.RecordFinalized.WaitOne(10000))
                {
                    // we (re)connect the recordHolder to the new Record
                    holder.RecordFinalized.Reset();
                    holder.Record = newRecord;
                }
                else
                {
                    throw new Exception("Something very wrong happened.");
                }
            }
            newRecord.SetHolderForNewRecord(holder);

            Byte[] sourceBuffer = holder.GetCurrentBuffer(/*readIfNeeded*/true);
            OnReadRecordBuffer(sourceBuffer, newRecord);

            return newRecord;
        }

        private void RemoveLastUsed(LinkedListNode<RecordHolder> linkListNode)
        {
            if (linkListNode.List == mRecordsByLastUsed)
                mRecordsByLastUsed.Remove(linkListNode);
        }

        private void SetLastUsed(LinkedListNode<RecordHolder> linkListNode)
        {
            if (linkListNode.List == mRecordsByLastUsed)
            {
                if (linkListNode != mRecordsByLastUsed.Last)
                {
                    mRecordsByLastUsed.Remove(linkListNode);
                    mRecordsByLastUsed.AddLast(linkListNode);
                }
            }
            else
            {
                mRecordsByLastUsed.AddLast(linkListNode);
            }

            // for every table we use PERCENT_CACHED_RECORDS
            int limit = (int)(PERCENT_CACHED_RECORDS * mRecordCount);
            if (limit < MIN_CACHED_RECORDS) limit = MIN_CACHED_RECORDS;
            else if (limit > MAX_CACHED_RECORDS) limit = MAX_CACHED_RECORDS;

            var node = mRecordsByLastUsed.First;
            while (node != null && mRecordsByLastUsed.Count > limit)
            {
                RecordHolder recordHolder = node.Value;
                var nextNode = node.Next;

                node.Value.Hold = false;
                mRecordsByLastUsed.Remove(node);
#if DUMP_FINALIZE
                System.Diagnostics.Trace.WriteLine(this.mOriginalFile + " Remove last used Record #" + recordHolder.mRecordNo);
#endif
                node = nextNode;
            }
        }



        /// <summary>
        /// Note that this method is called from flush and from the record destructor in the garbage collector thread.
        /// </summary>
        /// <param name="recordHolder"></param>
        internal void InternalFillWriteBuffer(RecordHolder recordHolder, Record record)
        {
            Debug.Assert(record != null, "InternalFillWriteBuffer : record must be not null.");
            var newBuffer = new Byte[mRecordWidth];

            if (OnFillWriteBuffer(record, newBuffer))
            {
                recordHolder.SetNewBuffer(newBuffer);
            }
        }

        protected abstract Record OnCreateNewRecord(bool isNew, UInt32 recordNo);
        protected abstract void OnReadRecordBuffer(byte[] buffer, Record record);
        internal abstract bool OnFillWriteBuffer(Record record, Byte[] buffer);
        protected abstract void OnInitialize(Stream mStream);
        protected abstract void OnWriteHeader(Stream stream);

        internal virtual void OnWriteBufferModified(UInt32 recordNo, byte[] oldBuffer, byte[] newBuffer)
        {
        }

        public virtual object NewRecord()
        {
            var recordNo = mRecordCount;
            mRecordCount += 1;
            var result = OnCreateNewRecord(/*isnew*/true, recordNo);

            var holder = new RecordHolder(this, recordNo, result);
            var linkedListNode = new LinkedListNode<RecordHolder>(holder);

            result.SetHolderForNewRecord(holder);

            mRecordsByRecordNo[recordNo] = linkedListNode;
            SetLastUsed(linkedListNode);
#if DUMP_FINALIZE
                System.Diagnostics.Trace.WriteLine(this.mOriginalFile + " Add Record #" + recordNo);
#endif


            return result;
        }

        internal ColumnDefinition GetColumnByName(string columnName)
        {
            for (int i = 0; i < mColumns.Count; i++)
            {
                if (mColumns[i].mColumnName == columnName)
                {
                    return mColumns[i];
                }
            }
            // not found try again with case off
            for (int i = 0; i < mColumns.Count; i++)
            {
                if (string.Compare(mColumns[i].mColumnName, columnName, true) == 0)
                {
                    return mColumns[i];
                }
            }
            return null;
        }

        static public T Get<T>(string filepath, OpenFileMode mode, int bufferSize)
            where T : ClusteredFile, new()
        {
            var fi = new System.IO.FileInfo(filepath);
            // in case the file is readonly we assume users expect us to force the OpenReadOnly mode.
            if (fi.Exists && (fi.Attributes & FileAttributes.ReadOnly) != 0) mode = OpenFileMode.OpenReadOnly;
            ClusteredFile clusteredFile = null;
            if (!mOpenFiles.TryGetValue(fi.FullName, out clusteredFile))
            {
                clusteredFile = new T();
                mOpenFiles.Add(fi.FullName, clusteredFile);
            }
            FileMode fm;
            FileAccess fa;
            FileShare fs;
            switch (mode)
            {
                case OpenFileMode.OpenOrCreate:
                    fm = FileMode.OpenOrCreate;
                    fa = FileAccess.ReadWrite;
                    fs = FileShare.Read;
                    break;
                case OpenFileMode.OpenReadOnly:
                    fm = FileMode.Open;
                    fa = FileAccess.Read;
                    fs = FileShare.ReadWrite;
                    break;
                default: // OpenFileMode.OpenReadWrite:
                    fm = FileMode.Open;
                    fa = FileAccess.ReadWrite;
                    fs = FileShare.Read;
                    break;
            }
            clusteredFile.InternalAddReader(fi, fm, fa, fs, bufferSize);
            return (T)clusteredFile;
        }

        private void InternalAddReader(FileInfo fi, FileMode fm, FileAccess fa, FileShare fs, int bufferSize)
        {
            if (mStream == null)
            {
                mStream = new FileStream(fi.FullName, fm, fa, fs, bufferSize);
                mFileMode = fm;
                mFileAccess = fa;
                mFileShare = fs;

                //if (encoding == null) encoding = System.Text.Encoding.ASCII;
                //mEncoding = encoding;
                mOriginalFile = fi.FullName;
                mCurrentRecordNoPosition = UInt32.MaxValue;
                mRecordsByRecordNo = new Dictionary<UInt32, LinkedListNode<RecordHolder>>();
#if DUMP_FINALIZE
                System.Diagnostics.Trace.WriteLine(this.mOriginalFile + "Clear Records");
#endif
                mRecordsByLastUsed = new LinkedList<RecordHolder>();
            }
            else
            {
                // todo check that the access make sense.
            }
            mReaderCount++;
        }

        public void StopReading()
        {
            mReaderCount--;
            if (mReaderCount == 0)
            {
                Dispose(true);
            }
        }

        public int ReaderCount
        {
            get
            {
                return mReaderCount;
            }
        }

        protected void WriteHeader()
        {
            mStream.Position = 0;
            OnWriteHeader(mStream);
            mCurrentRecordNoPosition = UInt32.MaxValue;
        }

        protected void Initialize()
        {
            OnInitialize(mStream);
        }

        internal void EmptyTable()
        {
            var node = mRecordsByLastUsed.First;

            while (node != null && mRecordsByLastUsed.Count > 0)
            {
                RecordHolder recordHolder = node.Value;
                var nextNode = node.Next;
                recordHolder.Save();
                node = nextNode;
            }
#if DUMP_FINALIZE
            System.Diagnostics.Trace.WriteLine(this.mOriginalFile + " Clear Records");
#endif

            mRecordsByRecordNo.Clear();
            mRecordsByLastUsed.Clear();
            mRecordCount = 0;
            mStream.SetLength(mHeaderWidth);
        }


        internal void InternalReadRecordFromDisk(UInt32 recordNo, Byte[] buffer)
        {
            Seek(recordNo);
            mStream.Read(buffer, 0, mRecordWidth);
            mCurrentRecordNoPosition = recordNo + 1;
#if DUMP_DISK_ACTIVITY
            System.Diagnostics.Debug.WriteLine(mOriginalFile + " ReadFromDisk Record #" + recordNo);
            System.Diagnostics.Debug.WriteLine(Utils.HexDump(buffer));
#endif
        }

        internal void InternalSaveRecordToDisk(UInt32 recordNo, Byte[] buffer)
        {
            Seek(recordNo);
            mStream.Write(buffer, 0, mRecordWidth);
            mCurrentRecordNoPosition = recordNo + 1;
#if DUMP_DISK_ACTIVITY
            System.Diagnostics.Debug.WriteLine(mOriginalFile + " SaveToDisk Record #" + recordNo);
            System.Diagnostics.Debug.WriteLine(Utils.HexDump(buffer));
#endif
        }

    }
}

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)

Share

About the Author

Pascal Ganaye
Software Developer (Senior)
United Kingdom United Kingdom
I am a French programmer.
These days I spend most of my time with the .NET framework, JavaScript and html.

| Advertise | Privacy | Mobile
Web04 | 2.8.140922.1 | Last Updated 10 Jun 2009
Article Copyright 2009 by Pascal Ganaye
Everything else Copyright © CodeProject, 1999-2014
Terms of Service
Layout: fixed | fluid