Click here to Skip to main content
15,892,199 members
Articles / Programming Languages / C#

Sharepoint Crawler Rules Problem and Workarround

Rate me:
Please Sign up or sign in to vote.
3.33/5 (2 votes)
14 Oct 2008CPOL2 min read 32.6K   120   11  
Sharepoint Craw Rules doesnt work in some case, especially excluding data.
namespace PSS.SearchVisibilitySetter
{
    using Microsoft.SharePoint;
    using System;
    using System.Collections.Generic;
    using System.ComponentModel;
    using System.Drawing;
    using System.IO;
    using System.Text.RegularExpressions;
    using System.Windows.Forms;

    public class Form1 : Form
    {
        private Button buttonGo;
        private Button buttonSaveLogs;
        private IContainer components = null;
        private const string exclude = "exclude ";
        private const string include = "include ";
        private Label label1;
        private Label label2;
        private List<Rule> m_rules;
        private RichTextBox richTextBoxLogs;
        private SaveFileDialog saveFileDialog1;
        private TabControl tabControl1;
        private TabPage tabPage1;
        private TabPage tabPage2;
        private TextBox textBoxRules;
        private TextBox textBoxSiteUrl;

        public Form1()
        {
            this.InitializeComponent();
        }

        private void buttonGo_Click(object sender, EventArgs e)
        {
            this.ClearLog();
            SPSite site = null;
            SPWeb rootWeb = null;
            try
            {
                try
                {
                    site = new SPSite(this.textBoxSiteUrl.Text);
                    rootWeb = site.RootWeb;
                    this.GetRules();
                    this.ProcessWeb(rootWeb);
                    MessageBox.Show("Done!", "Success", MessageBoxButtons.OK, MessageBoxIcon.Asterisk);
                }
                catch (Exception exception)
                {
                    MessageBox.Show("Error: " + exception.Message, "Error", MessageBoxButtons.OK, MessageBoxIcon.Hand);
                }
            }
            finally
            {
                if (rootWeb != null)
                {
                    rootWeb.Close();
                }
                if (site != null)
                {
                    site.Close();
                }
            }
        }

        private void buttonSaveLogs_Click(object sender, EventArgs e)
        {
            this.saveFileDialog1.Filter = "Log files (*.log)|*.log|Text files (*.txt)|*.txt|All files (*.*)|*.*";
            if (this.saveFileDialog1.ShowDialog() == DialogResult.OK)
            {
                File.WriteAllText(this.saveFileDialog1.FileName, this.richTextBoxLogs.Text.Replace("\n", Environment.NewLine));
            }
        }

        private void ClearLog()
        {
            this.richTextBoxLogs.Text = "";
        }

        protected override void Dispose(bool disposing)
        {
            if (disposing && (this.components != null))
            {
                this.components.Dispose();
            }
            base.Dispose(disposing);
        }

        private void GetRules()
        {
            this.m_rules = new List<Rule>();
            string[] strArray = Regex.Split(this.textBoxRules.Text.ToLower(), @"\n");
            foreach (string str in strArray)
            {
                string str2 = str.Trim(new char[] { '\r', '\n', ' ', '/' });
                if (!string.IsNullOrEmpty(str2))
                {
                    Rule item = new Rule();
                    if (str2.StartsWith("include "))
                    {
                        item.RegExRule = str2.Substring("include ".Length).Replace("*", ".*");
                        item.Include = true;
                    }
                    if (str2.StartsWith("exclude "))
                    {
                        item.RegExRule = str2.Substring("exclude ".Length).Replace("*", ".*");
                        item.Include = false;
                    }
                    this.m_rules.Add(item);
                }
            }
        }

        private void InitializeComponent()
        {
            this.label1 = new Label();
            this.textBoxSiteUrl = new TextBox();
            this.textBoxRules = new TextBox();
            this.label2 = new Label();
            this.buttonGo = new Button();
            this.tabControl1 = new TabControl();
            this.tabPage1 = new TabPage();
            this.tabPage2 = new TabPage();
            this.buttonSaveLogs = new Button();
            this.saveFileDialog1 = new SaveFileDialog();
            this.richTextBoxLogs = new RichTextBox();
            this.tabControl1.SuspendLayout();
            this.tabPage1.SuspendLayout();
            this.tabPage2.SuspendLayout();
            base.SuspendLayout();
            this.label1.AutoSize = true;
            this.label1.Location = new Point(6, 0x12);
            this.label1.Name = "label1";
            this.label1.Size = new Size(0x61, 13);
            this.label1.TabIndex = 0;
            this.label1.Text = "Site collection root:";
            this.textBoxSiteUrl.Location = new Point(0x6d, 15);
            this.textBoxSiteUrl.Name = "textBoxSiteUrl";
            this.textBoxSiteUrl.Size = new Size(520, 20);
            this.textBoxSiteUrl.TabIndex = 0;
            this.textBoxSiteUrl.Text = "http://localhost:90";
            this.textBoxRules.Location = new Point(0x6d, 0x29);
            this.textBoxRules.Multiline = true;
            this.textBoxRules.Name = "textBoxRules";
            this.textBoxRules.Size = new Size(520, 0x83);
            this.textBoxRules.TabIndex = 1;
            this.label2.AutoSize = true;
            this.label2.Location = new Point(0x44, 0x2c);
            this.label2.Name = "label2";
            this.label2.Size = new Size(0x25, 13);
            this.label2.TabIndex = 3;
            this.label2.Text = "Rules:";
            this.buttonGo.Location = new Point(0x22a, 0xb2);
            this.buttonGo.Name = "buttonGo";
            this.buttonGo.Size = new Size(0x4b, 0x17);
            this.buttonGo.TabIndex = 2;
            this.buttonGo.Text = "Go!";
            this.buttonGo.UseVisualStyleBackColor = true;
            this.buttonGo.Click += new EventHandler(this.buttonGo_Click);
            this.tabControl1.Controls.Add(this.tabPage1);
            this.tabControl1.Controls.Add(this.tabPage2);
            this.tabControl1.Dock = DockStyle.Fill;
            this.tabControl1.Location = new Point(0, 0);
            this.tabControl1.Name = "tabControl1";
            this.tabControl1.SelectedIndex = 0;
            this.tabControl1.Size = new Size(0x287, 0xf4);
            this.tabControl1.TabIndex = 3;
            this.tabPage1.Controls.Add(this.label1);
            this.tabPage1.Controls.Add(this.buttonGo);
            this.tabPage1.Controls.Add(this.textBoxSiteUrl);
            this.tabPage1.Controls.Add(this.label2);
            this.tabPage1.Controls.Add(this.textBoxRules);
            this.tabPage1.Location = new Point(4, 0x16);
            this.tabPage1.Name = "tabPage1";
            this.tabPage1.Padding = new Padding(3);
            this.tabPage1.Size = new Size(0x27f, 0xda);
            this.tabPage1.TabIndex = 0;
            this.tabPage1.Text = "Main";
            this.tabPage1.UseVisualStyleBackColor = true;
            this.tabPage2.Controls.Add(this.richTextBoxLogs);
            this.tabPage2.Controls.Add(this.buttonSaveLogs);
            this.tabPage2.Location = new Point(4, 0x16);
            this.tabPage2.Name = "tabPage2";
            this.tabPage2.Padding = new Padding(3);
            this.tabPage2.Size = new Size(0x27f, 0xda);
            this.tabPage2.TabIndex = 1;
            this.tabPage2.Text = "Logs";
            this.tabPage2.UseVisualStyleBackColor = true;
            this.buttonSaveLogs.Location = new Point(0x22c, 0xbb);
            this.buttonSaveLogs.Name = "buttonSaveLogs";
            this.buttonSaveLogs.Size = new Size(0x4b, 0x17);
            this.buttonSaveLogs.TabIndex = 1;
            this.buttonSaveLogs.Text = "Save logs";
            this.buttonSaveLogs.UseVisualStyleBackColor = true;
            this.buttonSaveLogs.Click += new EventHandler(this.buttonSaveLogs_Click);
            this.richTextBoxLogs.Location = new Point(6, 6);
            this.richTextBoxLogs.Name = "richTextBoxLogs";
            this.richTextBoxLogs.Size = new Size(0x271, 0xaf);
            this.richTextBoxLogs.TabIndex = 2;
            this.richTextBoxLogs.Text = "";
            base.AutoScaleDimensions = new SizeF(6f, 13f);
            base.AutoScaleMode = AutoScaleMode.Font;
            base.ClientSize = new Size(0x287, 0xf4);
            base.Controls.Add(this.tabControl1);
            base.Name = "Form1";
            this.Text = "Search Results Visibility Setter";
            this.tabControl1.ResumeLayout(false);
            this.tabPage1.ResumeLayout(false);
            this.tabPage1.PerformLayout();
            this.tabPage2.ResumeLayout(false);
            base.ResumeLayout(false);
        }

        private void Log(string message, bool error)
        {
            string text;
            if (error)
            {
                text = this.richTextBoxLogs.Text;
                this.richTextBoxLogs.Text = text + DateTime.Now.ToString("dd/MM/yyyy hh:mm:ss") + ": --> " + message + "\n\n";
            }
            else
            {
                text = this.richTextBoxLogs.Text;
                this.richTextBoxLogs.Text = text + DateTime.Now.ToString("dd/MM/yyyy hh:mm:ss") + ": " + message + "\n\n";
            }
        }

        private void ProcessWeb(SPWeb web)
        {
            if (!web.IsRootWeb)
            {
                this.SetCrawlVisibility(web, !this.ShouldBeHidden(web.Url), web.Url);
            }
            foreach (SPList list in web.Lists)
            {
                if (!list.Hidden)
                {
                    string url = list.ParentWeb.Url.Trim(new char[] { '/' }) + "/" + list.RootFolder.Url;
                    this.SetCrawlVisibility(list, !this.ShouldBeHidden(url), url);
                }
            }
            foreach (SPWeb web2 in web.Webs)
            {
                this.ProcessWeb(web2);
            }
        }

        private void SetCrawlVisibility(SPList list, bool visible, string url)
        {
            try
            {
               
                list.NoCrawl = !visible;
                list.Update();
                if (visible)
                {
                    this.Log("Include object SPList " + url + " in search results", false);
                }
                else
                {
                    this.Log("Exclude object SPList " + url + " from search results", false);
                }
            }
            catch (Exception exception)
            {
                this.Log(exception.Message, true);
            }
        }

        private void SetCrawlVisibility(SPWeb web, bool visible, string url)
        {
            try
            {
                web.NoCrawl = false;
                web.Update();
                if (visible)
                {
                    this.Log("Include object SPWeb " + url + " in search results", false);
                }
                else
                {
                    this.Log("Exclude object SPWeb " + url + " from search results", false);
                }
            }
            catch (Exception exception)
            {
                this.Log(exception.Message, true);
            }
        }

        private bool ShouldBeHidden(string url)
        {
            string input = url.ToLower().Trim(new char[] { '/' }) + "/";
            foreach (Rule rule in this.m_rules)
            {
                Regex regex = new Regex(rule.RegExRule);
                if (regex.Match(input).Success)
                {
                    if (rule.Include)
                    {
                        return false;
                    }
                    return true;
                }
            }
            return true;
        }
    }
}

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)


Written By
Team Leader BELBIM INC.
Turkey Turkey
This member has not yet provided a Biography. Assume it's interesting and varied, and probably something to do with programming.

Comments and Discussions