diff --git a/dBASE.NET/Dbf.cs b/dBASE.NET/Dbf.cs index d6479f6..7931ccf 100644 --- a/dBASE.NET/Dbf.cs +++ b/dBASE.NET/Dbf.cs @@ -1,5 +1,4 @@ -namespace dBASE.NET -{ +namespace dBASE.NET { using System; using System.Collections.Generic; using System.IO; @@ -9,15 +8,17 @@ /// The Dbf class encapsulated a dBASE table (.dbf) file, allowing /// reading from disk, writing to disk, enumerating fields and enumerating records. /// - public class Dbf - { + public class Dbf { private DbfHeader header; + /// Emanuele Bonin 24/03/2025 + /// path of DBF file + public string DBFPath; + /// /// Initializes a new instance of the . /// - public Dbf() - { + public Dbf() { header = DbfHeader.CreateHeader(DbfVersion.FoxBaseDBase3NoMemo); Fields = new List(); Records = new List(); @@ -28,8 +29,7 @@ public Dbf() /// /// Custom encoding. public Dbf(Encoding encoding) - : this() - { + : this() { Encoding = encoding ?? throw new ArgumentNullException(nameof(encoding)); } @@ -53,8 +53,7 @@ public Dbf(Encoding encoding) /// Creates a new with the same schema as the table. /// /// A with the same schema as the . - public DbfRecord CreateRecord() - { + public DbfRecord CreateRecord() { DbfRecord record = new DbfRecord(Fields); Records.Add(record); return record; @@ -64,20 +63,16 @@ public DbfRecord CreateRecord() /// Opens a DBF file, reads the contents that initialize the current instance, and then closes the file. /// /// The file to read. - public void Read(string path) - { + public void Read(string path) { + // Emanuele Bonin 24/03/2025 + DBFPath = path; // Open stream for reading. - using (FileStream baseStream = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read)) - { + using (FileStream baseStream = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read)) { string memoPath = GetMemoPath(path); - if (memoPath == null) - { + if (memoPath == null) { Read(baseStream); - } - else - { - using (FileStream memoStream = File.Open(memoPath, FileMode.Open, FileAccess.Read, FileShare.Read)) - { + } else { + using (FileStream memoStream = File.Open(memoPath, FileMode.Open, FileAccess.Read, FileShare.Read)) { Read(baseStream, memoStream); } } @@ -89,14 +84,11 @@ public void Read(string path) /// /// Stream with a database. /// Stream with a memo. - public void Read(Stream baseStream, Stream memoStream = null) - { - if (baseStream == null) - { + public void Read(Stream baseStream, Stream memoStream = null) { + if (baseStream == null) { throw new ArgumentNullException(nameof(baseStream)); } - if (!baseStream.CanSeek) - { + if (!baseStream.CanSeek) { throw new InvalidOperationException("The stream must provide positioning (support Seek method)."); } @@ -121,17 +113,31 @@ public void Read(Stream baseStream, Stream memoStream = null) /// /// The file to read. /// The version . If unknown specified, use current header version. - public void Write(string path, DbfVersion version = DbfVersion.Unknown) - { - if (version != DbfVersion.Unknown) - { + public void Write(string path, DbfVersion version = DbfVersion.Unknown) { + // Emanuele Bonin 24/03/2025 + DBFPath = path; + if (version != DbfVersion.Unknown) { header.Version = version; header = DbfHeader.CreateHeader(header.Version); } - using (FileStream stream = File.Open(path, FileMode.Create, FileAccess.Write)) - { - Write(stream, false); + using (FileStream stream = File.Open(path, FileMode.Create, FileAccess.Write)) { + bool Hasmemo = false; + for(int i=0; i < Fields.Count && !Hasmemo; i++) { + Hasmemo = Fields[i].Type == DbfFieldType.Memo; + } + header.HasMemoField = Hasmemo; + + if (header.HasMemoField) { + // Create Memo file header + string memoPath = Path.ChangeExtension(path, "fpt"); + using (FileStream memoStream = File.Open(memoPath, FileMode.Create, FileAccess.Write)) { + + VFPMemoHeader VFPMemoH = new VFPMemoHeader(); + VFPMemoH.Create(memoStream); + } + } + Write(stream, false); } } @@ -140,10 +146,8 @@ public void Write(string path, DbfVersion version = DbfVersion.Unknown) /// /// The output stream. /// The version . If unknown specified, use current header version. - public void Write(Stream stream, DbfVersion version = DbfVersion.Unknown) - { - if (version != DbfVersion.Unknown) - { + public void Write(Stream stream, DbfVersion version = DbfVersion.Unknown) { + if (version != DbfVersion.Unknown) { header.Version = version; header = DbfHeader.CreateHeader(header.Version); } @@ -151,32 +155,26 @@ public void Write(Stream stream, DbfVersion version = DbfVersion.Unknown) Write(stream, true); } - private void Write(Stream stream, bool leaveOpen) - { - using (BinaryWriter writer = new BinaryWriter(stream, Encoding, leaveOpen)) - { + private void Write(Stream stream, bool leaveOpen) { + using (BinaryWriter writer = new BinaryWriter(stream, Encoding, leaveOpen)) { header.Write(writer, Fields, Records); WriteFields(writer); WriteRecords(writer); } } - private static byte[] ReadMemos(Stream stream) - { - if (stream == null) - { + private static byte[] ReadMemos(Stream stream) { + if (stream == null) { throw new ArgumentNullException(nameof(stream)); } - using (MemoryStream ms = new MemoryStream()) - { + using (MemoryStream ms = new MemoryStream()) { stream.CopyTo(ms); return ms.ToArray(); } } - private void ReadHeader(BinaryReader reader) - { + private void ReadHeader(BinaryReader reader) { // Peek at version number, then try to read correct version header. byte versionByte = reader.ReadByte(); reader.BaseStream.Seek(0, SeekOrigin.Begin); @@ -185,13 +183,11 @@ private void ReadHeader(BinaryReader reader) header.Read(reader); } - private void ReadFields(BinaryReader reader) - { + private void ReadFields(BinaryReader reader) { Fields.Clear(); // Fields are terminated by 0x0d char. - while (reader.PeekChar() != 0x0d) - { + while (reader.PeekChar() != 0x0d) { Fields.Add(new DbfField(reader, Encoding)); } @@ -199,36 +195,42 @@ private void ReadFields(BinaryReader reader) reader.ReadByte(); } - private void ReadRecords(BinaryReader reader, byte[] memoData) - { + private void ReadRecords(BinaryReader reader, byte[] memoData) { Records.Clear(); // Records are terminated by 0x1a char (officially), or EOF (also seen). - while (reader.PeekChar() != 0x1a && reader.PeekChar() != -1) - { - try - { + while (reader.PeekChar() != 0x1a && reader.PeekChar() != -1) { + try { Records.Add(new DbfRecord(reader, header, Fields, memoData, Encoding)); - } - catch (EndOfStreamException) { } + } catch (EndOfStreamException) { } } } - private void WriteFields(BinaryWriter writer) - { - foreach (DbfField field in Fields) - { - field.Write(writer, Encoding); + private void WriteFields(BinaryWriter writer) { + int Displacement = 0; + foreach (DbfField field in Fields) { + field.Write(writer, Encoding, Displacement); + Displacement += field.Length; } // Write field descriptor array terminator. - writer.Write((byte)0x0d); + writer.Write((byte)0x0d); + + // Emanuele Bonin 22/03/2025 + // For visualFoxPro DBF Table there are other 263 bytes to add to the header + // that is the path to the dbc that belong the table (all 0x00 for no databases) + bool isVFP = header.Version == DbfVersion.VisualFoxPro || header.Version == DbfVersion.VisualFoxProWithAutoIncrement; + if (isVFP) { + for (int i = 0; i < 263; i++) writer.Write((byte)0); + } + } - private void WriteRecords(BinaryWriter writer) - { - foreach (DbfRecord record in Records) - { + private void WriteRecords(BinaryWriter writer) { + + foreach (DbfRecord record in Records) { + // Emanuele Bonin 24/04/2025 + record.ParentDbf = this; record.Write(writer, Encoding); } @@ -236,14 +238,11 @@ private void WriteRecords(BinaryWriter writer) writer.Write((byte)0x1a); } - private static string GetMemoPath(string basePath) - { + private static string GetMemoPath(string basePath) { string memoPath = Path.ChangeExtension(basePath, "fpt"); - if (!File.Exists(memoPath)) - { + if (!File.Exists(memoPath)) { memoPath = Path.ChangeExtension(basePath, "dbt"); - if (!File.Exists(memoPath)) - { + if (!File.Exists(memoPath)) { return null; } } diff --git a/dBASE.NET/Dbf3Header.cs b/dBASE.NET/Dbf3Header.cs index b9e8e44..87e4406 100644 --- a/dBASE.NET/Dbf3Header.cs +++ b/dBASE.NET/Dbf3Header.cs @@ -26,10 +26,14 @@ internal override void Read(BinaryReader reader) internal override void Write(BinaryWriter writer, List fields, List records) { this.LastUpdate = DateTime.Now; - // Header length = header fields (32b ytes) - // + 32 bytes for each field - // + field descriptor array terminator (1 byte) - this.HeaderLength = (ushort)(32 + fields.Count * 32 + 1); + // Header length = header fields (32b ytes) + // + 32 bytes for each field + // + field descriptor array terminator (1 byte) + // Emanuele Bonin 22/03/2025 + // For visualFoxPro DBF Table there are other 263 bytes to add to the header + // that is the path to the dbc that belong the table (all 0x00 for no databases) + bool isVFP = this.Version == DbfVersion.VisualFoxPro || this.Version == DbfVersion.VisualFoxProWithAutoIncrement; + this.HeaderLength = (ushort)(32 + fields.Count * 32 + 1 + (isVFP ? 263 : 0)); this.NumRecords = (uint)records.Count; this.RecordLength = 1; foreach (DbfField field in fields) @@ -37,14 +41,32 @@ internal override void Write(BinaryWriter writer, List fields, List f.Type == DbfFieldType.Memo)) { + HasMemoField = true; + tableFlag |= 0x02; + } + + writer.Write((byte)Version); // 0x00 Version + writer.Write((byte)(LastUpdate.Year - 1900)); // 0x01 AA + writer.Write((byte)(LastUpdate.Month)); // 0x02 MM + writer.Write((byte)(LastUpdate.Day)); // 0x03 DD + writer.Write(NumRecords); // 0x04 - 0x007 Number of records + writer.Write(HeaderLength); // 0x08 - 0x09 Position of first data record + writer.Write(RecordLength); // 0x0A - 0x0B Length of one data record including delete flag + for (int i = 0; i < 16; i++) writer.Write((byte)0); // 0x0C - 0x1B Reserved + // Visual foxpro + writer.Write((byte)tableFlag); // 0x1C Table flags + // Values: + // 0x01 file has a structural .cdx + // 0x02 file has a Memo field (.fpt file) + // 0x04 file is a database (.dbc) + // This byte can contain the sum of any of the above values. + // For example, the value 0x03 indicates the table has a structural .cdx and a Memo field. + for (int i = 0; i < 3; i++) writer.Write((byte)0); } } } diff --git a/dBASE.NET/DbfField.cs b/dBASE.NET/DbfField.cs index 25c59fc..8d334b2 100644 --- a/dBASE.NET/DbfField.cs +++ b/dBASE.NET/DbfField.cs @@ -87,7 +87,7 @@ internal DbfField(BinaryReader reader, Encoding encoding) reader.ReadBytes(8); } - internal void Write(BinaryWriter writer, Encoding encoding) + internal void Write(BinaryWriter writer, Encoding encoding, int Displacement = 0) { // Pad field name with 0-bytes, then save it. string name = this.Name; @@ -107,7 +107,10 @@ internal void Write(BinaryWriter writer, Encoding encoding) } writer.Write((char)Type); - writer.Write((uint)0); // 4 reserved bytes: Field data address in memory. + + writer.Write((uint)Displacement); // 4 reserved bytes: Field data address in memory. + // Displacement of field in record + writer.Write(Length); writer.Write(Precision); writer.Write((ushort)0); // 2 reserved byte. diff --git a/dBASE.NET/DbfHeader.cs b/dBASE.NET/DbfHeader.cs index 191502d..c0ec4e8 100644 --- a/dBASE.NET/DbfHeader.cs +++ b/dBASE.NET/DbfHeader.cs @@ -38,7 +38,13 @@ public abstract class DbfHeader /// public ushort RecordLength { get; set; } - public static DbfHeader CreateHeader(DbfVersion version) + /// + /// Emanuele Bonin 24/03/2025 + /// Has a memo field ? (VisualFoxPro) + /// + public bool HasMemoField { get; set; } + + public static DbfHeader CreateHeader(DbfVersion version) { DbfHeader header; switch(version) diff --git a/dBASE.NET/DbfRecord.cs b/dBASE.NET/DbfRecord.cs index 551a0ab..f5ba947 100644 --- a/dBASE.NET/DbfRecord.cs +++ b/dBASE.NET/DbfRecord.cs @@ -1,8 +1,8 @@ -namespace dBASE.NET -{ +namespace dBASE.NET { using dBASE.NET.Encoders; using System; using System.Collections.Generic; + using System.Dynamic; using System.IO; using System.Linq; using System.Text; @@ -11,15 +11,15 @@ /// DbfRecord encapsulates a record in a .dbf file. It contains an array with /// data (as an Object) for each field. /// - public class DbfRecord - { + public class DbfRecord { private const string defaultSeparator = ","; private const string defaultMask = "{name}={value}"; + public Dbf ParentDbf; + private List fields; - internal DbfRecord(BinaryReader reader, DbfHeader header, List fields, byte[] memoData, Encoding encoding) - { + internal DbfRecord(BinaryReader reader, DbfHeader header, List fields, byte[] memoData, Encoding encoding) { this.fields = fields; Data = new List(); @@ -34,8 +34,7 @@ internal DbfRecord(BinaryReader reader, DbfHeader header, List fields, // Read data for each field. int offset = 0; - foreach (DbfField field in fields) - { + foreach (DbfField field in fields) { // Copy bytes from record buffer into field buffer. byte[] buffer = new byte[field.Length]; Array.Copy(row, offset, buffer, 0, field.Length); @@ -49,8 +48,7 @@ internal DbfRecord(BinaryReader reader, DbfHeader header, List fields, /// /// Create an empty record. /// - internal DbfRecord(List fields) - { + internal DbfRecord(List fields) { this.fields = fields; Data = new List(); foreach (DbfField field in fields) Data.Add(null); @@ -60,20 +58,16 @@ internal DbfRecord(List fields) public object this[int index] => Data[index]; - public object this[string name] - { - get - { + public object this[string name] { + get { int index = fields.FindIndex(x => x.Name.Equals(name)); if (index == -1) return null; return Data[index]; } } - public object this[DbfField field] - { - get - { + public object this[DbfField field] { + get { int index = fields.IndexOf(field); if (index == -1) return null; return Data[index]; @@ -84,8 +78,7 @@ public object this[DbfField field] /// Returns a string that represents the current object. /// /// A string that represents the current object. - public override string ToString() - { + public override string ToString() { return ToString(defaultSeparator, defaultMask); } @@ -94,8 +87,7 @@ public override string ToString() /// /// Custom separator. /// A string that represents the current object with custom separator. - public string ToString(string separator) - { + public string ToString(string separator) { return ToString(separator, defaultMask); } @@ -108,30 +100,91 @@ public string ToString(string separator) /// e.g., "{name}={value}", where {name} is the mask for the field name, and {value} is the mask for the value. /// /// A string that represents the current object with custom separator and mask. - public string ToString(string separator, string mask) - { + public string ToString(string separator, string mask) { separator = separator ?? defaultSeparator; mask = (mask ?? defaultMask).Replace("{name}", "{0}").Replace("{value}", "{1}"); return string.Join(separator, fields.Select(z => string.Format(mask, z.Name, this[z]))); } - internal void Write(BinaryWriter writer, Encoding encoding) - { + internal void Write(BinaryWriter writer, Encoding encoding) { + // Emanuele Bonin 22/03/2025 + // VFP MemoFile + string MemoFile; + bool HasMemo = false; + FileStream stream = null; + BinaryWriter Memowriter = null; + BinaryReader Memoreader = null; + int UsedBlocks = 0, BlockSize = 0, FreeBlockPointer = 0; // Write marker (always "not deleted") writer.Write((byte)0x20); int index = 0; - foreach (DbfField field in fields) - { + HasMemo = fields.Any(f => f.Type == DbfFieldType.Memo); + if (HasMemo) { + // Emanuele Bonin 22/03/2025 + // VFP MemoFile + MemoFile = Path.ChangeExtension(ParentDbf.DBFPath, "fpt"); + + stream = File.Open(MemoFile, FileMode.OpenOrCreate, FileAccess.ReadWrite); + Memowriter = new BinaryWriter(stream, Encoding.ASCII); + Memoreader = new BinaryReader(stream, Encoding.ASCII); + + + + // Read 32-bit integer as big endian + byte[] bytes = Memoreader.ReadBytes(4); // 0x00 - 0x03 next free block + Array.Reverse(bytes); + FreeBlockPointer = BitConverter.ToInt32(bytes, 0); + Memoreader.BaseStream.Seek(6, SeekOrigin.Begin); // 0x06 - 0x07 block size in bytes + bytes = Memoreader.ReadBytes(2); + Array.Reverse(bytes); + BlockSize = BitConverter.ToInt16(bytes, 0); + + } + // https://www.vfphelp.com/help/_5WN12PC0N.htm + foreach (DbfField field in fields) { IEncoder encoder = EncoderFactory.GetEncoder(field.Type); + byte[] buffer = encoder.Encode(field, Data[index], encoding); + if (field.Type == DbfFieldType.Memo) { + + if (!buffer.All(b => b == 0x20)) { + + // Write on memo file + // and get the memo position + Memowriter.Seek(FreeBlockPointer * BlockSize, SeekOrigin.Begin); + Memowriter.Write(BitConverter.GetBytes((int)1).Reverse().ToArray()); // 0x00 - 0x03 Block signature Big Endian + // (indicates the type of data in the block) + // 0 – picture (picture field type) + // 1 – text (memo field type) + + Memowriter.Write(BitConverter.GetBytes((int)buffer.Length).Reverse().ToArray()); // Length of memo(in bytes) Big Endian + Memowriter.Write(buffer); + UsedBlocks = (int)Math.Ceiling(buffer.Length / (decimal)BlockSize); + // Fill rest of the used blocks with 0x00 + for (int i = 0; i < UsedBlocks * BlockSize - buffer.Length; i++) { + Memowriter.Write((byte)0x00); + } + buffer = BitConverter.GetBytes((int)FreeBlockPointer); + FreeBlockPointer = FreeBlockPointer + UsedBlocks; + } else { + UsedBlocks = 0; + buffer = BitConverter.GetBytes((int)0); + } + + } if (buffer.Length > field.Length) throw new ArgumentOutOfRangeException(nameof(buffer.Length), buffer.Length, "Buffer length has exceeded length of the field."); - writer.Write(buffer); index++; } + if (HasMemo) { + Memowriter.Seek(0, SeekOrigin.Begin); + Memowriter.Write(BitConverter.GetBytes((int)FreeBlockPointer).Reverse().ToArray()); + Memowriter.Close(); + Memoreader.Close(); + } } } } diff --git a/dBASE.NET/Encoders/IntegerEncoder.cs b/dBASE.NET/Encoders/IntegerEncoder.cs index 55cd5e8..c463859 100644 --- a/dBASE.NET/Encoders/IntegerEncoder.cs +++ b/dBASE.NET/Encoders/IntegerEncoder.cs @@ -1,10 +1,8 @@ -namespace dBASE.NET.Encoders -{ +namespace dBASE.NET.Encoders { using System; using System.Text; - public class IntegerEncoder : IEncoder - { + public class IntegerEncoder : IEncoder { private static IntegerEncoder instance; private IntegerEncoder() { } @@ -12,16 +10,29 @@ private IntegerEncoder() { } public static IntegerEncoder Instance => instance ?? (instance = new IntegerEncoder()); /// - public byte[] Encode(DbfField field, object data, Encoding encoding) - { + public byte[] Encode(DbfField field, object data, Encoding encoding) { int value = 0; - if (data != null) value = (int)data; + if (data != null) { + if (data is decimal) { + // Handle decimal to int conversion + // This is a workaround for the issue with decimal to int conversion + // in .NET where it throws an exception if the decimal is not a whole number + // and the value is not a valid integer. + value = Convert.ToInt32((decimal)data); + + } else if (data is string) { + if (!Int32.TryParse(data.ToString(), out value)) value = 0; + } else if (data == null || data == DBNull.Value) { + value = 0; + } else { + value = Convert.ToInt32(data); + } + } return BitConverter.GetBytes(value); } /// - public object Decode(byte[] buffer, byte[] memoData, Encoding encoding) - { + public object Decode(byte[] buffer, byte[] memoData, Encoding encoding) { return BitConverter.ToInt32(buffer, 0); } } diff --git a/dBASE.NET/Encoders/MemoEncoder.cs b/dBASE.NET/Encoders/MemoEncoder.cs index 0c04b6d..b7b7589 100644 --- a/dBASE.NET/Encoders/MemoEncoder.cs +++ b/dBASE.NET/Encoders/MemoEncoder.cs @@ -1,6 +1,7 @@ namespace dBASE.NET.Encoders { using System; + using System.Collections.Generic; using System.Text; internal class MemoEncoder : IEncoder @@ -10,11 +11,37 @@ internal class MemoEncoder : IEncoder private MemoEncoder() { } public static MemoEncoder Instance => instance ?? (instance = new MemoEncoder()); + + // cach different length bytes (for performance) + Dictionary buffers = new Dictionary(); + + private byte[] GetBuffer(int length) { + if (!buffers.TryGetValue(length, out var bytes)) { + var s = new string(' ', length); + bytes = Encoding.ASCII.GetBytes(s); + buffers.Add(length, bytes); + } + return (byte[])bytes.Clone(); + } /// public byte[] Encode(DbfField field, object data, Encoding encoding) { - return null; + // Input data maybe various: int, string, whatever. + string res = data?.ToString(); + if (string.IsNullOrEmpty(res)) { + res = field.DefaultValue; + } + // Emanuele Bonin + // 24/03/2025 + int BufferLen = res.Length; + + // consider multibyte should truncate or padding after GetBytes (11 bytes) + var buffer = GetBuffer(BufferLen); + var bytes = encoding.GetBytes(res); + Array.Copy(bytes, buffer, Math.Min(bytes.Length, BufferLen)); + + return buffer; } /// diff --git a/dBASE.NET/VFPMemoHeader.cs b/dBASE.NET/VFPMemoHeader.cs new file mode 100644 index 0000000..4d2f02a --- /dev/null +++ b/dBASE.NET/VFPMemoHeader.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using System.Dynamic; +using System.IO; +using System.Linq; +using System.Numerics; +using System.Runtime.InteropServices.ComTypes; +using System.Text; +using System.Threading.Tasks; + +namespace dBASE.NET { + public class VFPMemoHeader { + private int BlockSize = 64; + + public void Create(Stream sw) { + using (BinaryWriter writer = new BinaryWriter(sw, Encoding.ASCII, false)) { + /* + Byte offset Description + 00 – 03 Location of next free block (Big Endian) + 04 – 05 Unused + 06 – 07 Block size(bytes per block) (Big Endian) + 08 – 511 Unused + */ + + writer.Write(BitConverter.GetBytes(0x08).Reverse().ToArray()); // Location of next free block (8 = 512/BlockSize) in Big Endian + writer.Write((short)0x00); // Unused + writer.Write(BitConverter.GetBytes((short)BlockSize).Reverse().ToArray()); // Block size in Big Endian + for (int i = 0; i < 504; i++) { + writer.Write((byte)0x00); // Unused + } // 0x08 - 0x1FF + // 0x200 First free block + } + } + } +}