Click here to Skip to main content
Click here to Skip to main content
Articles » Database » NoSQL » General » Downloads
 
Add your own
alternative version
Go to top

RavenDB - An Introduction

, , 28 Apr 2010
An introduction to RavenDB - a new open source .NET document database using .NET 4.0 and VS 2010
Raven_DB.zip
RavenDB
.gitignore
ayende-open-source.snk
build
default.ps1
psake.ps1
psake_ext.ps1
Raven.Client.Tests
Document
Properties
Shard
Transactions
Raven.Client
Client
Document
Properties
Shard
ShardStrategy
ShardAccess
ShardResolution
ShardSelection
Util
Raven.Database
Data
Defaults
Exceptions
Extensions
Indexing
Json
Linq
PrivateExtensions
Properties
Storage
Tasks
Raven.FileStorage
Properties
Raven.Importer
Properties
Raven.Sample.ShardClient
Properties
Raven.Sample.SimpleClient
Properties
Raven.Scenarios
Properties
Scenarios
AddDocument_AddIndex_QueryIndexForDoc.saz
AfterTooManyErrorsDocumentIndexingForTheIndexWillBeDisabled.saz
AfterTooManyErrorsIndexWillBeDisabled.saz
BatchAndGetDocuments.saz
BatchQueryById.saz
BrowseIndex.saz
CanGenerateIdentityValues.saz
CanGetMetadataWhenLookingAtAllDocs.saz
CanGetMetadataWhenLookingAtAnIndex.saz
CannotDeleteStaticWithoutAccruateETag.saz
CanPutAndDeleteDocumentWithETag.saz
CanUseCollectionNamesInIndexes.saz
DeleteStatic.saz
DeletingDocumentThatDoesNotExistsAlwaysWorks.saz
DeletingWithBadETagWillFail.saz
IndexingErrorsWillShowUpInTheStats.saz
IndexingFunctionWithMultipleResultsPerSourceDocument.saz
PatchDocument.saz
PutAndGetAttachment.saz
PutAndGetDocument.saz
PutAndGetDocumentWithMetadataViaHeaders.saz
PutAndGetIndex.saz
PuttingDocumentUsingTransaction.saz
QueryMapReduceIndex.saz
QueryUsingIndex.saz
RecordHeadersForStatics.saz
SelectFieldsFromIndex.saz
TryingToUpdateAttachmentWithBadETagWillFail.saz
TryingToUpdateDocumentWithBadETagWillFail.saz
TryingToUpdateDocumentWithNonGuidETagWillFail.saz
UpdateAttachemtnUsingETag.saz
UpdatingDocsWillUpdateMapReduceResult.saz
WhenDeletingDocsWillUpdateMapReduceIndex.saz
WillDeleteDocumentFromIndex.saz
WillSupport304.saz
WillSupport304ForAttachments.saz
Raven.Server
Defaults
default.json
PowerShellProvider
Properties
Responders
WebUI
css
smoothness
images
ui-bg_flat_0_aaaaaa_40x100.png
ui-bg_flat_75_ffffff_40x100.png
ui-bg_glass_55_fbf9ee_1x400.png
ui-bg_glass_65_ffffff_1x400.png
ui-bg_glass_75_dadada_1x400.png
ui-bg_glass_75_e6e6e6_1x400.png
ui-bg_glass_95_fef1ec_1x400.png
ui-bg_highlight-soft_75_cccccc_1x100.png
ui-icons_222222_256x240.png
ui-icons_2e83ff_256x240.png
ui-icons_454545_256x240.png
ui-icons_888888_256x240.png
ui-icons_cd0a0a_256x240.png
favicon.ico
images
ajax-loader.gif
bgR.png
c1_i1.png
c2_i2.png
c2_i3.png
c2_i4.png
c2_i5.png
c2_i6.png
c3_i7.png
footer_bg.png
footerContainerBgR.png
header_bg.png
jsonEditor
doc.gif
docNode.gif
docNodeLast.gif
docNodeLastFirst.gif
folder.gif
folderNode.gif
folderNodeFirst.gif
folderNodeLast.gif
folderNodeLastFirst.gif
folderNodeOpen.gif
folderNodeOpenFirst.gif
folderNodeOpenLast.gif
folderNodeOpenLastFirst.gif
folderOpen.gif
vertLine.gif
logo.png
midBox1.png
midBox2Arrow.png
sideBarArrow.png
sideBarListBoxBottom.png
sideBarListBoxTop.png
submit.png
topNavSep.png
js
jstree
themes
default
dot_for_ie.gif
icons.png
throbber.gif
icons
txt.png
rdb.jsonEditor
JSONTemplates
Raven.Tests
Indexes
Linq
Patching
Properties
Storage
Transactions
Views
Raven.Tryouts
Properties
RavenDB.suo
Release
SharedLibs
Esent.Interop.dll
Esent.Interop.pdb
ICSharpCode.NRefactory.dll
ICSharpCode.SharpZipLib.dll
log4net.dll
Lucene.Net.dll
Lucene.Net.pdb
Newtonsoft.Json.dll
Rhino.Mocks.dll
SgmlReaderDll.dll
SgmlReaderDll.pdb
System.Management.Automation.dll
xunit.dll
xunit.extensions.dll
Tools
ILMerge.exe
xUnit
xunit.console.exe
xunit.dll
xunit.extensions.dll
xunit.gui.exe
xunit.installer.exe
XUnit.NAntTasks.dll
xunit.runner.msbuild.dll
xunit.runner.resharper.dll
xunit.runner.tdnet.dll
xunit.runner.utility.dll
zip.exe
Utilities
Binaries
ILMerge.exe
Newtonsoft.Json.dll
Raven.DefaultDatabase.Creator.exe
Raven.DefaultDatabase.Creator.pdb
Raven.Merger.exe
Raven.Merger.pdb
SgmlReaderDll.dll
SgmlReaderDll.pdb
Raven.DefaultDatabase.Creator
Properties
Raven.Merger
Properties
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using log4net;
using Lucene.Net.Analysis.Standard;
using Lucene.Net.Documents;
using Lucene.Net.Index;
using Lucene.Net.QueryParsers;
using Lucene.Net.Search;
using Lucene.Net.Store;
using Raven.Database.Data;
using Raven.Database.Extensions;
using Raven.Database.Linq;
using Raven.Database.Storage;

namespace Raven.Database.Indexing
{
	/// <summary>
	/// 	This is a thread safe, single instance for a particular index.
	/// </summary>
	public abstract class Index : IDisposable
	{
		private readonly Directory directory;
		protected readonly ILog log = LogManager.GetLogger(typeof (Index));
		protected readonly string name;
		protected readonly IndexDefinition indexDefinition;
		private CurrentIndexSearcher searcher;

		protected Index(Directory directory, string name,IndexDefinition indexDefinition)
		{
			this.name = name;
			this.indexDefinition = indexDefinition;
			log.DebugFormat("Creating index for {0}", name);
			this.directory = directory;
			searcher = new CurrentIndexSearcher
			{
				Searcher = new IndexSearcher(directory)
			};
		}

		#region IDisposable Members

		public void Dispose()
		{
			searcher.Searcher.Close();
			directory.Close();
		}

		#endregion

		public IEnumerable<IndexQueryResult> Query(IndexQuery indexQuery)
		{
			using (searcher.Use())
			{
				var search = ExecuteQuery(indexQuery, GetLuceneQuery(indexQuery));
				indexQuery.TotalSize.Value = search.Length();
				var previousDocuments = new HashSet<string>();
				for (var i = indexQuery.Start; i < search.Length() && (i - indexQuery.Start) < indexQuery.PageSize; i++)
				{
					var document = search.Doc(i);
					if (IsDuplicateDocument(document, indexQuery.FieldsToFetch, previousDocuments))
						continue;
					yield return RetrieveDocument(document, indexQuery.FieldsToFetch);
				}
			}
		}

		private Hits ExecuteQuery(IndexQuery indexQuery, Query luceneQuery)
		{
			Hits search;
			if (indexQuery.SortedFields != null)
			{
				var sort = new Sort(indexQuery.SortedFields.Select(x => x.ToLuceneSortField()).ToArray());
				search = searcher.Searcher.Search(luceneQuery, sort);
			}
			else
			{
				search = searcher.Searcher.Search(luceneQuery);
			}
			return search;
		}

		private Query GetLuceneQuery(IndexQuery indexQuery)
		{
			var query = indexQuery.Query;
			Query luceneQuery;
			if(string.IsNullOrEmpty(query))
			{
				log.DebugFormat("Issuing query on index {0} for all documents", name);
				luceneQuery = new MatchAllDocsQuery();	
			}
			else
			{
				log.DebugFormat("Issuing query on index {0} for: {1}", name, query);
				luceneQuery = new QueryParser("", new StandardAnalyzer()).Parse(query);
			}
			return luceneQuery;
		}

		private static bool IsDuplicateDocument(Document document, ICollection<string> fieldsToFetch, ISet<string> previousDocuments)
		{
			if (fieldsToFetch != null && fieldsToFetch.Count > 1)
				return false;
			return previousDocuments.Add(document.Get("__document_id")) == false;
		}

		public abstract void IndexDocuments(AbstractViewGenerator viewGenerator, IEnumerable<object> documents,
		                                    WorkContext context,
		                                    DocumentStorageActions actions);

		protected abstract IndexQueryResult RetrieveDocument(Document document, string[] fieldsToFetch);

		protected void Write(Func<IndexWriter, bool> action)
		{
			var indexWriter = new IndexWriter(directory, new StandardAnalyzer());
			bool shouldRcreateSearcher;
			try
			{
				shouldRcreateSearcher = action(indexWriter);
			}
			finally
			{
				indexWriter.Close();
			}
			if (shouldRcreateSearcher)
				RecreateSearcher();
		}


		protected IEnumerable<object> RobustEnumeration(IEnumerable<object> input, IndexingFunc func,
		                                                DocumentStorageActions actions, WorkContext context)
		{
			var wrapped = new StatefulEnumerableWrapper<dynamic>(input.GetEnumerator());
			IEnumerator<object> en = func(wrapped).GetEnumerator();
			do
			{
				var moveSuccessful = MoveNext(en, wrapped, context, actions);
				if (moveSuccessful == false)
					yield break;
				if (moveSuccessful == true)
					yield return en.Current;
				else
					en = func(wrapped).GetEnumerator();
			} while (true);
		}

		private bool? MoveNext(IEnumerator en, StatefulEnumerableWrapper<object> innerEnumerator, WorkContext context,
		                       DocumentStorageActions actions)
		{
			try
			{
				actions.IncrementIndexingAttempt();
				var moveNext = en.MoveNext();
				if (moveNext == false)
					actions.DecrementIndexingAttempt();
				return moveNext;
			}
			catch (Exception e)
			{
				actions.IncrementIndexingFailure();
				context.AddError(name,
				                 TryGetDocKey(innerEnumerator.Current),
				                 e.Message
					);
				log.WarnFormat(e, "Failed to execute indexing function on {0} on {1}", name,
				               GetDocId(innerEnumerator));
			}
			return null;
		}

		private static string TryGetDocKey(object current)
		{
			var dic = current as DynamicJsonObject;
			if (dic == null)
				return null;
			var value = dic.GetValue("__document_id");
			if (value == null)
				return null;
			return value.ToString();
		}

		private static object GetDocId(StatefulEnumerableWrapper<object> currentInnerEnumerator)
		{
			var dictionary = currentInnerEnumerator.Current as IDictionary<string, object>;
			if (dictionary == null)
				return null;
			object docId;
			dictionary.TryGetValue("__document_id", out docId);
			return docId;
		}

		private void RecreateSearcher()
		{
			using (searcher.Use())
			{
				searcher.MarkForDispoal();
				searcher = new CurrentIndexSearcher
				{
					Searcher = new IndexSearcher(directory)
				};
				Thread.MemoryBarrier(); // force other threads to see this write
			}
		}

		public abstract void Remove(string[] keys, WorkContext context);

		#region Nested type: CurrentIndexSearcher

		private class CurrentIndexSearcher
		{
			private bool shouldDisposeWhenThereAreNoUsages;
			private int useCount;
			public IndexSearcher Searcher { get; set; }


			public IDisposable Use()
			{
				Interlocked.Increment(ref useCount);
				return new CleanUp(this);
			}

			public void MarkForDispoal()
			{
				shouldDisposeWhenThereAreNoUsages = true;
			}

			#region Nested type: CleanUp

			private class CleanUp : IDisposable
			{
				private readonly CurrentIndexSearcher parent;

				public CleanUp(CurrentIndexSearcher parent)
				{
					this.parent = parent;
				}

				#region IDisposable Members

				public void Dispose()
				{
					var uses = Interlocked.Decrement(ref parent.useCount);
					if (parent.shouldDisposeWhenThereAreNoUsages && uses == 0)
						parent.Searcher.Close();
				}

				#endregion
			}

			#endregion
		}

		#endregion
	}
}

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)

Share

About the Authors

No Biography provided

Paul B.

United States United States
I've been a software developer since 1996 and have enjoyed C# since 2003. I have a Bachelor's degree in Computer Science and for some reason, a Master's degree in Business Administration. I currently do software development contracting/consulting.

You may also be interested in...

| Advertise | Privacy | Mobile
Web04 | 2.8.140916.1 | Last Updated 28 Apr 2010
Article Copyright 2010 by Ayende @ Rahien, Paul B.
Everything else Copyright © CodeProject, 1999-2014
Terms of Service
Layout: fixed | fluid