Read a large text file to a list

PointList = System.IO.File.ReadAllLines(MyFileName).ToList

Quote:

Philippe,

Here's what I have:

a text file containing a (huge) list of 3d points (X,Y,Z) which I call T1, T2, T3 etc....:

-3153.61743,-6115.18164,-3289.31226
-3256.75195,-6075.70801,-3550.83057
-3775.00977,-5870.36035,-3817.91870
-3890.03223,-5819.92871,-3797.58984
-4134.60156,-5724.16309,-3651.39648
....

For each point (called T) I need to calculate if any of the other points (called x1, x2 etc) are
within a cone drawn from the vector of T to an infinite point on that vector. In its 'shadow' if you like.
At the end I need a list of all such points.

Here's what I do:
1) Read the X,Y,Z into three lists (of integers - that's close enough)
2) Calculate the range of each point and its two spherical bearings
3) Sort the list by range so that the closest to origin are at the top of the list
4) Create seperate lists of the X, Y, Z, Range, Bearing#1, Bearing#2 value of every point
5) Run a loop a bit like this:

For every T
For every X (which is T+1)
Calculate vectors from range, bearing#1, bearing #2 for X to see if its in the cone vector
If yes, add to a List of Y
Next X
Next T
Write the Y list to file.


Here's a few things I've tried to speed things up:
1) I determine which 3d quadrant each X is in - if its not in the same as the current T I dont run the vector calcs
2) I tried writing the Y to disk using a Streamwriter, every 10, 100, 1000 points of Y but this was very slow.
3) I tried decreasing the size of the lists using List.RemoveAt(0) each time the T loop was completed, made no difference to speed
4) To be able to monitor progress (albeit slow!) the loop lives in a Background worker with a progress bar and % completed label.

I'm sure there's a clever/fast way to do this, but it escapes me!

using System;
using System.Collections.Generic;
using System.IO;
using System.Windows.Media.Media3D;

namespace Stuff
{
  class Program
  {
    static void Main(string[] args){ }
    class Thing : IComparable<Thing>
    {
      private static readonly Vector3DConverter V3Parser = new Vector3DConverter();
      public Thing(Vector3D raw)
      {
        _Raw = raw;
        // Describe the point in Spherical Polar Coordinates
        R = _Raw.Length; 
        // For now, assume R isn't 0
        Theta = Math.Acos(raw.Z / R);
        Phi = Math.Atan2(raw.Y, raw.X);
        // Quadrant: anticlockwise from -pi
        Q = Phi < -Math.PI / 2 ? 0 :
            Phi < 0 ? 1 :
            Phi < Math.PI / 2 ? 2 :
            3;
      }
      public Thing(string textual)
        : this((Vector3D)V3Parser.ConvertFromString(textual))
      { }
      private readonly Vector3D _Raw;
      public Vector3D Raw { get { return _Raw; } }

      public readonly double R;
      public readonly double Theta;
      public readonly double Phi;
      public readonly int Q;

      #region IComparable<Thing> Members
      public int CompareTo(Thing other)
      {
        if (other == null)
          throw new ArgumentNullException("other");
        return R.CompareTo(other.R);
      }
      #endregion

      public override string ToString()
      {
        return V3Parser.ConvertToString(_Raw);
      }

      public static Vector3D operator -(Thing thing1, Thing thing2)
      {
        return thing1._Raw - thing2._Raw;
      }
    }
    static void ConeProblem(string MyFileName)
    {
      List<Thing> AllThings = new List<Thing>();
      foreach (string line in File.ReadLines(MyFileName))
      {
        AllThings.Add(new Thing(line));
      }
      AllThings.Sort();
      List<Thing> SelectedThings = new List<Thing>();
      for (int i = 0; i < AllThings.Count; i++)
      {
        Thing ti = AllThings[i];
        for (int j = i + 1; j < AllThings.Count; j++)
        {
          Thing tj = AllThings[j];
          if (IsInConeVector(ti, tj))
            SelectedThings.Add(tj);
        }
      }
      // Now write out the SelectedThings
    }
    static bool IsInConeVector(Thing t1, Thing t2)
    {
      if (t1.Q != t2.Q)
        return false;
      // You still need to do this (HARD) part.
      return true;
    }
  }
}

Import System
Import System.Collections.Generic
Import System.IO
Import System.Windows.Media.Media3D

Private Class Thing
 Implements IComparable(Of Thing)
 Private Shared ReadOnly V3Parser As Vector3DConverter = New Vector3DConverter
 Public Sub New(ByVla textual As String)
  Me.New(DirectCast(Thing.V3Parser.ConvertFromString(textual), Vector3D))
 End Sub

 Public Sub New(ByVal raw as Vector3D)
  Me._Raw = raw
  ' Describe the point in Spherical Polar Coordinates
  Me.R = raw.Length
  ' For now, assume R isn't 0
  Me.Theta = Math.Acos(raw.Z / Me.R)
  Me.Phi = Math.Atan2(raw, Y, raw.X)
  ' Quadrant: anticlockwise from -pi
  Me.Q = If((Me.Phi < -Math.PI/2), 0,
         If((Me.Phi < 0), 1, 
         If((Me.Phi < Math.PI/2), 2, 3)))
 End Sub

 Public Function CompareTo(ByVal other As Thing) As Integer
  If (other Is Nothing) Then
   Throw New ArgumentNullException("other")
  End If
  Return Me.R.CompareTo(other.R)
 End Function

 Public Shared Operator -(ByVal thing1 As Thing, ByVal thing2 as Thing) As Vector3D
  Return (thing1._Raw - thing2._Raw)
 End Function

 Public Overrides Function ToString() As String
  return Thing.V3Parser.ConvertToString(Me._Raw)
 End Function

 Public ReadOnly Property Raw As Vector3D
  Get
   Return Me._Raw
  End Get
 End Property

 Private ReadOnly _Raw As Vector3D
 Public ReadOnly Phi As Double
 Public ReadOnly Q As Integer
 Public ReadOnly R As Double
 Public ReadOnly Theta As Double
End Class

Private Shared Sub ConeProblem(ByVal MyFileName As String)
 Dim AllThings as New List(Of Thing)
 Dim line as String
 For Each line In File.ReadLines(MyFileName)
  AllThings.Add(New Thing(line))
 Next
 AllThings.Sort()
 Dim SelectedThings As New List(Of Thing)
 Dim i As Integer
 For i = 0 To AllThings.Count - 1
  Dim ti As Thing = AllThings.Item(i)
  Dim j As Integer
  For j = (i + 1) To AllThings.Count - 1
   Dim tj As Thing = AllThings.Item(j)
   If Program.IsInConeVector(ti, tj) Then
    SelectedThings.Add(tj)
   End If
  Next j
 Next i
 ' Now write out the SelectedThings
End Sub

Private Shared Function IsInConeVector(ByVal t1 As Thing, ByVal t2 As Thing) As Boolean
 If (t1.Q <> t2.Q) Then
  Return False
 End If
 ' You still need to do this (HARD) part.
 Return True
End Function