Ultra large text file parsing(size more that 100GB)

''' <summary>
''' size smaller than 1GB
''' </summary>
''' <param name="LogFile">Original plant text file path of the blast output file.</param>
''' <returns></returns>
''' <remarks></remarks>
Public Shared Function TryParse(LogFile As String) As v228
   Call Console.WriteLine("Regular Expression parsing blast output...")

   Using p As Microsoft.VisualBasic.ConsoleProcessBar = New ConsoleProcessBar
      Call p.Start()

      Dim SourceText As String = IO.File.ReadAllText(LogFile) 'LogFile.ReadUltraLargeTextFile(System.Text.Encoding.UTF8)
      Dim Sections As String() = (From matche As Match
                                  In Regex.Matches(SourceText, BLAST_QUERY_HIT_SECTION, RegexOptions.Singleline + RegexOptions.IgnoreCase)
                                  Select matche.Value).ToArray

      Call Console.WriteLine("Parsing job done!")

      Dim Sw As Stopwatch = Stopwatch.StartNew
#If DEBUG Then
      Dim LQuery = (From Line As String In Sections Let query = v228.Query.TryParse(Line) Select query Order By query.QueryName Ascending).ToArray
#Else
      Dim LQuery = (From Line As String In Sections.AsParallel Let query = v228.Query.TryParse(Line) Select query Order By query.QueryName Ascending).ToArray
#End If
      Dim BLASTOutput As v228 = New v228 With {.FilePath = LogFile & ".xml", .Queries = LQuery}
      Console.WriteLine("BLASTOutput file loaded: {0}ms", Sw.ElapsedMilliseconds)

      Return BLASTOutput
   End Using
End Function

''' <summary>
''' It seems 786MB possibly is the up bound of the Utf8.GetString function.
''' </summary>
''' <remarks></remarks>
Const CHUNK_SIZE As Long = 1024 * 1024 * 786
Const BLAST_QUERY_HIT_SECTION As String = "Query=.+?Effective search space used: \d+"

''' <summary>
''' Dealing with the file size large than 2GB
''' </summary>
''' <param name="LogFile"></param>
''' <returns></returns>
''' <remarks></remarks>
Public Shared Function TryParseUltraLarge(LogFile As String, Optional Encoding As System.Text.Encoding = Nothing) As v228
    Call Console.WriteLine("Regular Expression parsing blast output...")

    'The default text encoding of the blast log is utf8
    If Encoding Is Nothing Then Encoding = System.Text.Encoding.UTF8

    Using p As Microsoft.VisualBasic.ConsoleProcessBar = New ConsoleProcessBar
       Call p.Start()

       Dim TextReader As IO.FileStream = New IO.FileStream(LogFile, IO.FileMode.Open)
       Dim ChunkBuffer As Byte() = New Byte(CHUNK_SIZE - 1) {}
       Dim LastIndex As String = ""
       'Dim Sections As List(Of String) = New List(Of String)
       Dim SectionChunkBuffer As List(Of String) = New List(Of String)

       Do While TextReader.Position < TextReader.Length
          Dim Delta As Integer = TextReader.Length - TextReader.Position

          If Delta < CHUNK_SIZE Then ChunkBuffer = New Byte(Delta - 1) {}

          Call TextReader.Read(ChunkBuffer, 0, ChunkBuffer.Count - 1)

          Dim SourceText As String = Encoding.GetString(ChunkBuffer)

          If Not String.IsNullOrEmpty(LastIndex) Then
             SourceText = LastIndex & SourceText
          End If

          Dim i_LastIndex As Integer = InStrRev(SourceText, "Effective search space used:")
          If i_LastIndex = -1 Then  '当前区间之中没有一个完整的Section
             LastIndex &= SourceText
             Continue Do
          Else
             i_LastIndex += 42

             If Not i_LastIndex >= Len(SourceText) Then
                LastIndex = Mid(SourceText, i_LastIndex)  'There are some text in the last of this chunk is the part of the section in the next chunk.
             Else
                LastIndex = ""
             End If
             Call SectionChunkBuffer.Add(SourceText)
          End If

          'This part of the code is non-parallel

          'Dim SectionsTempChunk = (From matche As Match
          '                         In Regex.Matches(SourceText, BLAST_QUERY_HIT_SECTION, RegexOptions.Singleline + RegexOptions.IgnoreCase)
          '                         Select matche.Value).ToArray

          'If SectionsTempChunk.IsNullOrEmpty Then
          '    LastIndex &= SourceText
          '    Continue Do
          'Else
          '    Call Sections.AddRange(SectionsTempChunk)
          'End If

          'LastIndex = SectionsTempChunk.Last()
          'Dim Last_idx As Integer = InStr(SourceText, LastIndex) + Len(LastIndex) + 1
          'If Not Last_idx >= Len(SourceText) Then
          '    LastIndex = Mid(SourceText, Last_idx)  'There are some text in the last of this chunk is the part of the section in the next chunk.
          'Else
          '    LastIndex = ""
          'End If
      Loop

      Call Console.WriteLine("Loading job done, start to regex parsing!")

      'The regular expression parsing function just single thread, here using parallel to parsing the cache data can speed up the regular expression parsing job when dealing with the ultra large text file.
       Dim Sections As String() = (From strLine As String 
                                   In SectionChunkBuffer.AsParallel
                                   Select (From matche As Match
                                           In Regex.Matches(strLine, BLAST_QUERY_HIT_SECTION, RegexOptions.Singleline + RegexOptions.IgnoreCase)
                                           Select matche.Value).ToArray).ToArray.MatrixToVector

       Call Console.WriteLine("Parsing job done!")
       '#Const DEBUG = 1
       Dim Sw As Stopwatch = Stopwatch.StartNew
#If DEBUG Then
       Dim LQuery = (From Line As String In Sections Let query = v228.Query.TryParse(Line) Select query Order By query.QueryName Ascending).ToArray
#Else
       Dim LQuery = (From Line As String In Sections.AsParallel Let query = v228.Query.TryParse(Line) Select query Order By query.QueryName Ascending).ToArray
#End If
       Dim BLASTOutput As v228 = New v228 With {.FilePath = LogFile & ".xml", .Queries = LQuery}
       Console.WriteLine("BLASTOutput file loaded: {0}ms", Sw.ElapsedMilliseconds)

       Return BLASTOutput
   End Using
End Function