LINQ To Google Image and Google Groups

using System;
using System.IO;
using System.Collections.Generic;

using System.Text;
using System.Text.RegularExpressions;

using System.Web;
using System.Net;
using System.Linq;
using System.Linq.Expressions;

using MChen.Linq.GoogleSearch.Common;

namespace MChen.Linq.GoogleSearch
{
    internal abstract class Searcher<T, S> : ExpressionVisitor<S>,
            IQueryable<T>, IOrderedQueryable<T> 
            where S : QueryInfo 
            where T : Result
        {
            protected S      _info;

            protected abstract Regex  _regex { get; }

            protected abstract Result CreateResult(Match match);

            protected IList<T> RequestForPage(int start)
            {
                //send web request and collect results
                string url = _info.GetUrl(start);
                Console.WriteLine(url);
                WebRequest request = WebRequest.Create(url);
                WebResponse response = request.GetResponse();

                string content = null;
                using (StreamReader sr = new StreamReader(response.GetResponseStream()))
                {
                    content = sr.ReadToEnd();
                }

                //Console.WriteLine("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
                //Console.WriteLine(content);
                //Console.WriteLine("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~");

                //parse result
                List<T> list = new List<T>();
                if (content != null)
                {
                    int cnt = 1;
                    foreach (Match m in _regex.Matches(content))
                    {
                        Result res = CreateResult(m);
                        res.Rank = start + cnt ++;
                        list.Add((T)res);
                    }
                }
                return (IList<T>)list;
            }

            #region IQueryable<T> Members
            public IQueryable<X> CreateQuery<X>(System.Linq.Expressions.Expression expression)
            {
                if (expression == null)
                {
                    throw new ArgumentException("expression");
                }
                if (!typeof(IQueryable<X>).IsAssignableFrom(expression.Type))
                {
                    throw new ArgumentException("expression");
                }
                
                //handle projection
                if (!typeof(Result).IsAssignableFrom(typeof(X)))
                {
                    if (expression.NodeType == ExpressionType.Call
                        && ExpressionUtil.IsSequenceOperatorCall((MethodCallExpression)expression))
                    {
                        return new Projector<X, T>((MethodCallExpression)expression);
                    }
                    throw new NotSupportedException(
                              "Query not supported.");
                }

                Diagnostic.DebugExpressionTree(expression);
                _info = Visit(expression, _info);
                Console.WriteLine(_info);
                return (IQueryable<X>)this;
            }

            public TResult Execute<TResult>(System.Linq.Expressions.Expression expression)
            {
                throw new Exception("The method or operation is not implemented.");
            }

            #endregion

            #region IEnumerable<T> Members

            public IEnumerator<T> GetEnumerator()
            {
                int cnt = 0;
                //implementation of enumerator
                while (true)
                {
                    IList<T> batch = RequestForPage(cnt);
                    foreach (var img in batch)
                    {
                        cnt++;
                        yield return img;
                    }

                    //stop condition
                    if (batch.Count == 0) break;
                }
            }

            #endregion

            #region IEnumerable Members

            System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
            {
                return GetEnumerator();
            }

            #endregion

            #region IQueryable Members

            public IQueryable CreateQuery(System.Linq.Expressions.Expression expression)
            {
                throw new Exception("The method or operation is not implemented.");
            }

            public Type ElementType
            {
                get { return typeof(T); }
            }

            public object Execute(System.Linq.Expressions.Expression expression)
            {
                throw new Exception("The method or operation is not implemented.");
            }

            public System.Linq.Expressions.Expression Expression
            {
                get { return Expression.Constant(this); }
            }

            #endregion
    }
}