Update - Removed double lookup on dictionaries.

Author: ikelaiah

entries/ikelaiah/README.md

@@ -127,6 +127,10 @@ Iwan Kelaiah
   * Converting Float as String to Int was a bit slow, so resorted to a lookup instead. This saves 30-45 seconds.
   * Re-arranged `if` statements in two places. This saves 10-15 seconds x 2 = ~ 30 seconds saving.
 
+* 1.8
+  * Revision release - Sequential approach. 4-6 mins on my Inspiron 15 7510 laptop (a little improvement on speed).
+  * Removed double lookup on dictionaries.
+
 ## License
 
 This project is licensed under the MIT License - see the LICENSE.md file for details

entries/ikelaiah/src/weatherstation.pas

@@ -52,6 +52,7 @@   TWeatherStation = class
     lookupStrFloatToIntList: TValidTemperatureDictionary;
     procedure CreateLookupTemp;
     procedure ReadMeasurements;
+    procedure ReadMeasurementsBuf;
     procedure ParseStationAndTemp(const line: string);
     procedure AddCityTemperatureLG(const cityName: string; const newTemp: int64);
     procedure SortWeatherStationAndStats;
@@ -239,10 +240,10 @@ procedure TWeatherStation.AddCityTemperatureLG(const cityName: string;
   stat: PStat;
 begin
   // If city name esxists, modify temp as needed
-  if self.weatherDictionary.Contains(cityName) then
+  if self.weatherDictionary.TryGetValue(cityName, stat) then
   begin
     // Get the temp record
-    stat := self.weatherDictionary[cityName];
+    // stat := self.weatherDictionary[cityName];
 
     // Update min and max temps if needed
     // Re-arranged the if statement, to achieve minimal if checks.
@@ -309,11 +310,9 @@ procedure TWeatherStation.ParseStationAndTemp(const line: string);
     strFloatTemp := Copy(line, delimiterPos + 1, Length(line));
 
     // Using a lookup value speeds up 30-45 seconds
-    if self.lookupStrFloatToIntList.Contains(strFloatTemp) then
+    if self.lookupStrFloatToIntList.TryGetValue(strFloatTemp, parsedTemp) then
     begin
-      parsedTemp := self.lookupStrFloatToIntList[strFloatTemp];
-      self.AddCityTemperatureLG(Copy(line, 1, delimiterPos - 1),
-        parsedTemp);
+      self.AddCityTemperatureLG(Copy(line, 1, delimiterPos - 1), parsedTemp);
     end;
   end;
 end;
@@ -346,11 +345,99 @@ procedure TWeatherStation.ReadMeasurements;
   end;
 end;
 
+procedure TWeatherStation.ReadMeasurementsBuf;
+var
+  fileStream: TFileStream;
+  memStream: TMemoryStream;
+  streamReader: TStreamReader;
+  buffer: TBytes;
+  bytesRead, totalBytesRead, chunkSize, lineBreakPos, chunkIndex,
+  index, lineCount: int64;
+begin
+
+  chunksize := 536870912 * 1;
+
+  // Open the file for reading
+  fileStream := TFileStream.Create(self.fname, fmOpenRead);
+  SetLength(buffer, chunkSize);
+  try
+    memStream := TMemoryStream.Create;
+    try
+      totalBytesRead := 0;
+      chunkIndex := 0;
+      lineCount := 0;
+
+      // Read and parse chunks of data until EOF
+      while totalBytesRead < fileStream.Size do
+      begin
+        // Read more bytes and keep track on bytes read
+        bytesRead := fileStream.Read(buffer[0], chunkSize);
+        Inc(totalBytesRead, bytesRead);
+
+        // Find the position of the last newline character in the chunk
+        lineBreakPos := BytesRead;
+        while (lineBreakPos > 0) and (Buffer[lineBreakPos - 1] <> Ord(#10)) do
+          Dec(lineBreakPos);
+
+        { Now, must ensure that if the last byte read in the current chunk
+          is not a newline character, the file pointer is moved back to include
+          that byte and any preceding bytes of the partial line in the next
+          chunk's read operation.
+
+          Also, no need to update the BytesRead variable in this context because
+          it represents the actual number of bytes read from the file, including
+          any partial line that may have been included due to moving the file
+          pointer back.
+          Ref: https://www.freepascal.org/docs-html/rtl/classes/tstream.seek.html}
+        if lineBreakPos < bytesRead then
+          fileStream.Seek(-(bytesRead - lineBreakPos), soCurrent);
+        //{$IFDEF DEBUG}
+        // Do something with the chunk here
+        // Like counting line
+        for index := 0 to lineBreakPos - 1 do
+          if buffer[index] = Ord(#10) then
+            lineCount := lineCount + 1;
+        //{$ENDIF DEBUG}
+
+        // Use memory stream & stream reader
+        memStream.Write(buffer[0], lineBreakPos - 1);
+        memStream.Position:=0;
+        streamReader := TStreamReader.Create(memStream);
+        try
+          while not streamReader.EOF do
+          begin
+            // WriteLn(streamReader.ReadLine);
+            self.ParseStationAndTemp(streamReader.ReadLine);
+          end;
+        finally
+          streamReader.Free;
+        end;
+        //{$IFDEF DEBUG}
+        // Display user feedback
+        WriteLn('Line count: ', IntToStr(lineCount));
+        WriteLn('Chunk ', chunkIndex, ', Total bytes read:', IntToStr(totalBytesRead));
+        //{$ENDIF DEBUG}
+
+        //{$IFDEF DEBUG}
+        // Increase chunk index - a counter
+        Inc(chunkIndex);
+        //{$ENDIF DEBUG}
+      end;
+    finally
+      memStream.Free;
+    end;
+  finally
+    // Close the file
+    fileStream.Free;
+  end;
+end;
+
 // The main algorithm
 procedure TWeatherStation.ProcessMeasurements;
 begin
   self.CreateLookupTemp;
   self.ReadMeasurements;
+  //self.ReadMeasurementsBuf;
   self.SortWeatherStationAndStats;
   self.PrintSortedWeatherStationAndStats;
 end;