--- oup/current/Helper/LevelDB.pas 2007/03/23 00:59:01 120 +++ oup/current/Helper/LevelDB.pas 2007/03/30 00:00:10 137 @@ -1,63 +1,3 @@ -{$A8,B-,C+,D+,E-,F-,G+,H+,I+,J-,K-,L+,M-,N+,O+,P+,Q-,R-,S-,T-,U-,V+,W-,X+,Y+,Z1} -{$MINSTACKSIZE $00004000} -{$MAXSTACKSIZE $00100000} -{$IMAGEBASE $00400000} -{$APPTYPE GUI} -{$WARN SYMBOL_DEPRECATED ON} -{$WARN SYMBOL_LIBRARY ON} -{$WARN SYMBOL_PLATFORM ON} -{$WARN SYMBOL_EXPERIMENTAL ON} -{$WARN UNIT_LIBRARY ON} -{$WARN UNIT_PLATFORM ON} -{$WARN UNIT_DEPRECATED ON} -{$WARN UNIT_EXPERIMENTAL ON} -{$WARN HRESULT_COMPAT ON} -{$WARN HIDING_MEMBER ON} -{$WARN HIDDEN_VIRTUAL ON} -{$WARN GARBAGE ON} -{$WARN BOUNDS_ERROR ON} -{$WARN ZERO_NIL_COMPAT ON} -{$WARN STRING_CONST_TRUNCED ON} -{$WARN FOR_LOOP_VAR_VARPAR ON} -{$WARN TYPED_CONST_VARPAR ON} -{$WARN ASG_TO_TYPED_CONST ON} -{$WARN CASE_LABEL_RANGE ON} -{$WARN FOR_VARIABLE ON} -{$WARN CONSTRUCTING_ABSTRACT ON} -{$WARN COMPARISON_FALSE ON} -{$WARN COMPARISON_TRUE ON} -{$WARN COMPARING_SIGNED_UNSIGNED ON} -{$WARN COMBINING_SIGNED_UNSIGNED ON} -{$WARN UNSUPPORTED_CONSTRUCT ON} -{$WARN FILE_OPEN ON} -{$WARN FILE_OPEN_UNITSRC ON} -{$WARN BAD_GLOBAL_SYMBOL ON} -{$WARN DUPLICATE_CTOR_DTOR ON} -{$WARN INVALID_DIRECTIVE ON} -{$WARN PACKAGE_NO_LINK ON} -{$WARN PACKAGED_THREADVAR ON} -{$WARN IMPLICIT_IMPORT ON} -{$WARN HPPEMIT_IGNORED ON} -{$WARN NO_RETVAL ON} -{$WARN USE_BEFORE_DEF ON} -{$WARN FOR_LOOP_VAR_UNDEF ON} -{$WARN UNIT_NAME_MISMATCH ON} -{$WARN NO_CFG_FILE_FOUND ON} -{$WARN IMPLICIT_VARIANTS ON} -{$WARN UNICODE_TO_LOCALE ON} -{$WARN LOCALE_TO_UNICODE ON} -{$WARN IMAGEBASE_MULTIPLE ON} -{$WARN SUSPICIOUS_TYPECAST ON} -{$WARN PRIVATE_PROPACCESSOR ON} -{$WARN UNSAFE_TYPE OFF} -{$WARN UNSAFE_CODE OFF} -{$WARN UNSAFE_CAST OFF} -{$WARN OPTION_TRUNCATED ON} -{$WARN WIDECHAR_REDUCED ON} -{$WARN DUPLICATES_IGNORED ON} -{$WARN UNIT_INIT_SEQ ON} -{$WARN LOCAL_PINVOKE ON} -{$WARN MESSAGE_DIRECTIVE ON} unit LevelDB; interface uses @@ -125,11 +65,8 @@ var Stream_Body, Stream_Names: TMemoryStream; Stream_Dat, Stream_Raw, Stream_Sep: TFileStream; -// Data, rawdata: Tdata; BeginTime, FileTime: Double; Step: Integer; -// rawlist: TRawDataList; -// datlinks: TDatLinks; OniImage: TOniImage; LevelID: Integer; TimeFormat: TFormatSettings; @@ -150,6 +87,12 @@ var RawLinks: TRawDataList; DatFileStream, RawFileStream: TMemoryStream; + +// ########################### + datsum, linksum, rawsum: Int64; + freq: Int64; + tempticks1, tempticks2: Int64; +// ########################### const Steps: Byte = 3; @@ -211,15 +154,20 @@ begin begin ShowMessage('Couldn''t delete file. Aborting'); Exit; - end else if not DeleteFile(AnsiReplaceStr(Target, '.dat', '.raw')) then - begin - ShowMessage('Couldn''t delete file. Aborting'); - Exit; - end else if not DeleteFile(AnsiReplaceStr(Target, '.dat', '.sep')) then - begin - ShowMessage('Couldn''t delete file. Aborting'); - Exit; end; + if FileExists(AnsiReplaceStr(Target, '.dat', '.raw')) then + if not DeleteFile(AnsiReplaceStr(Target, '.dat', '.raw')) then + begin + ShowMessage('Couldn''t delete file. Aborting'); + Exit; + end; + if FileExists(AnsiReplaceStr(Target, '.dat', '.sep')) then + if Connection.DataOS in [DOS_WINDEMO, DOS_MAC, DOS_MACBETA] then + if not DeleteFile(AnsiReplaceStr(Target, '.dat', '.sep')) then + begin + ShowMessage('Couldn''t delete file. Aborting'); + Exit; + end; end else begin ShowMessage('Aborting'); Exit; @@ -335,6 +283,11 @@ begin lbl_estimation.Caption := 'Estimated finishing time: unknown'; Application.ProcessMessages; + QueryPerformanceFrequency(freq); + datsum := 0; + linksum := 0; + rawsum := 0; + FileTime := Time; for FileID := 0 to DatHeader.Files - 1 do begin @@ -343,12 +296,19 @@ begin FilesHeader[FileID].Extension[j] := FileInfo.Extension[4 - j]; if FileInfo.Size > 0 then begin + QueryPerformanceCounter(tempticks1); + FilesHeader[FileID].DataAddr := Stream_Body.Size + 8; DatFileStream := TMemoryStream.Create; Connection.LoadDatFile(FileID, TStream(DatFileStream)); - DatFileStream.Seek(4, soFromBeginning); + DatFileStream.Seek(0, soFromBeginning); + tempi := FileID * 256 + 1; + DatFileStream.Write(tempi, 4); DatFileStream.Write(LevelID, 4); + QueryPerformanceCounter(tempticks2); + datsum := datsum + (tempticks2 - tempticks1); + DatLinks := Connection.GetDatLinks(FileID); if Length(DatLinks) > 0 then begin @@ -363,6 +323,9 @@ begin end; end; + QueryPerformanceCounter(tempticks1); + linksum := linksum + (tempticks1 - tempticks2); + RawLinks := Connection.GetRawList(FileID); if Length(RawLinks) > 0 then begin @@ -371,108 +334,54 @@ begin if RawLinks[i].RawSize > 0 then begin RawFileStream := TMemoryStream.Create; - if UpperCase(fileinfo.Extension) = 'TXMP' then + Connection.LoadRawFile(FileID, RawLinks[i].SrcOffset, TStream(RawFileStream)); + RawFileStream.Seek(0, soFromBeginning); + if RawLinks[i].LocSep then begin - DatFileStream.Seek($88, soFromBeginning); - DatFileStream.Read(tempb, 1); - if (tempb and $01) > 0 then - begin - OniImage.Load(Connection.ConnectionID, FileID); - if not OniImage.GetMipMappedImage(TStream(RawFileStream)) then - begin - ShowMessage('MipMapping-Error'); - RawFileStream.Seek(0, soFromBeginning); - tempb := tempb and $FE; - DatFileStream.Seek($88, soFromBeginning); - DatFileStream.Write(tempb, 1); - OniImage.Load(Connection.ConnectionID, FileID); - OniImage.GetAsData(TStream(RawFileStream)); - end else - begin - // Change of Depth($89), Storetype ($90) - DatFileStream.Seek($89, soFromBeginning); - DatFileStream.Read(tempb, 1); - - DatFileStream.Seek($90, soFromBeginning); - DatFileStream.Read(tempb, 1); - Exit; - end; - end else - OniImage.GetAsData(TStream(RawFileStream)); + RawLinks[i].RawAddr := Stream_Sep.Size; + Stream_sep.CopyFrom(RawFileStream, RawFileStream.Size); end else begin - Connection.LoadRawFile(FileID, RawLinks[i].SrcOffset, TStream(RawFileStream)); + RawLinks[i].RawAddr := Stream_Raw.Size; + Stream_Raw.CopyFrom(RawFileStream, RawFileStream.Size); end; - RawFileStream.Seek(0, soFromBeginning); end else RawLinks[i].RawAddr := 0; + DatFileStream.Seek(RawLinks[i].SrcOffset, soFromBeginning); + DatFileStream.Write(RawLinks[i].RawAddr, 4); end; end; + DatFileStream.Seek(0, soFromBeginning); + Stream_Body.CopyFrom(DatFileStream, DatFileStream.Size); -{ - if rawlist[j].raw_size > 0 then - begin - if (UpperCase(fileinfo.Extension) = 'TXMP') and - ((Data[$88] and $01) > 0) then - begin - OniImage.LoadFromTXMP(Connection, i); - OniImage.GetMipMappedImage(rawdata); - rawlist[j].raw_size := OniImage.GetImageDataSize(True); - Data[$90] := $08; - Data[$89] := 32; -(* if data[$90]<>OniImage.StoreType then begin - data[$90]:=OniImage.StoreType; - data[$89]:=(data[$89] and $CF) or $20; - end; -*) end - else - begin - SetLength(rawdata, rawlist[j].raw_size); - OniDataConnection.LoadRawFile(i, rawlist[j].src_offset, @rawdata[0]); - end; - // data[$88]:=data[$88] and $FE; - - if rawlist[j].loc_sep then - begin - rawlist[j].raw_addr := Stream_Sep.Size; - Stream_Sep.Write(rawdata[0], Length(rawdata)); - end - else - begin - rawlist[j].raw_addr := Stream_Raw.Size; - Stream_Raw.Write(rawdata[0], Length(rawdata)); - end; - end - else - rawlist[j].raw_addr := 0; - Data[rawlist[j].src_offset + 0] := (rawlist[j].raw_addr) and $FF; - Data[rawlist[j].src_offset + 1] := (rawlist[j].raw_addr shr 8) and $FF; - Data[rawlist[j].src_offset + 2] := (rawlist[j].raw_addr shr 16) and $FF; - Data[rawlist[j].src_offset + 3] := (rawlist[j].raw_addr shr 24) and $FF; -} - Stream_Body.Write(Data[0], Length(Data)); - // + QueryPerformanceCounter(tempticks2); + rawsum := rawsum + (tempticks2 - tempticks1); end else - FilesHeader[i].DataAddr := 0; + FilesHeader[FileID].DataAddr := 0; if Length(fileinfo.Name) > 0 then begin - FilesHeader[i].NameAddr := Stream_Names.Size; + FilesHeader[FileID].NameAddr := Stream_Names.Size; temps := fileinfo.Extension + fileinfo.Name + Chr(0); Stream_Names.Write(temps[1], Length(temps)); end else - FilesHeader[i].NameAddr := 0; - FilesHeader[i].FileSize := fileinfo.Size; - FilesHeader[i].FileType := fileinfo.FileType; + FilesHeader[FileID].NameAddr := 0; + FilesHeader[FileID].FileSize := fileinfo.Size; + FilesHeader[FileID].FileType := fileinfo.FileType; - if ((i mod 10) = 0) and (i >= 100) then + if ((FileID mod 10) = 0) and (FileID >= 100) then lbl_estimation.Caption := 'Estimated time left: ' + TimeToStr( - (Time - FileTime) / i * (progress.Max - i + 1) * 1.1, TimeFormat ); - progress.Position := i + 1; - lbl_progress.Caption := 'Files done: ' + IntToStr(i + 1) + '/' + IntToStr(progress.Max); + (Time - FileTime) / FileID * (progress.Max - FileID + 1) * 1.1, TimeFormat ); + progress.Position := FileID + 1; + lbl_progress.Caption := 'Files done: ' + IntToStr(FileID + 1) + '/' + IntToStr(progress.Max); Application.ProcessMessages; end; + ShowMessage('AvgDats: ' + FloatToStr((datsum / progress.Max) / freq) + #13#10 + + 'AvgLinks: ' + FloatToStr((linksum / progress.Max) / freq) + #13#10 + + 'AvgRaws: ' + FloatToStr((rawsum / progress.Max) / freq) + ); + Stream_Dat.Write(DatHeader, SizeOf(DatHeader)); for i := 0 to High(FilesHeader) do Stream_Dat.Write(FilesHeader[i], SizeOf(FilesHeader[i])); @@ -484,6 +393,7 @@ begin DatHeader.DataSize := Stream_Body.Size; DatHeader.NamesSize := Stream_Names.Size; DatHeader.DataAddr := Stream_Dat.Size; + Stream_Body.Seek(0, soFromBeginning); Stream_Dat.CopyFrom(Stream_Body, Stream_Body.Size); DatHeader.NamesAddr := Stream_Dat.Size; @@ -497,7 +407,8 @@ begin Stream_Body.Free; Stream_Names.Free; Stream_Raw.Free; - if OniDataConnection.OSisMac then + + if Connection.DataOS in [DOS_WINDEMO, DOS_MAC, DOS_MACBETA] then Stream_Sep.Free; progress.Position := progress.Max; @@ -513,7 +424,7 @@ begin converting := False; - CloseDataConnection(DataConnections[conIndex]); +// CloseDataConnection(DataConnections[conIndex]); end; @@ -523,7 +434,7 @@ procedure TForm_LevelDB.HandleFile; var i: Byte; begin - for i := 1 to Length(ConvertHandlers) do +{ for i := 1 to Length(ConvertHandlers) do if UpperCase(ConvertHandlers[i].Ext) = UpperCase(ext) then if ConvertHandlers[i].needed then begin @@ -532,13 +443,16 @@ begin end else Break; -end; +}end; procedure TForm_LevelDB.CreateDatabase(Source, target: String); var + DataBase: TABSDatabase; +{ +var DataBase: TABSDatabase; Query: TABSQuery; MimeCoder: TStringFormat_MIME64; @@ -569,9 +483,10 @@ const else group_progress.Caption := 'Creating DB (FINISHED)'; end; +} begin - if CreateDataConnection(Source, ODB_Dat) = nil then +{ if CreateDataConnection(Source, ODB_Dat) = nil then begin ShowMessage('Could not connect to .dat-file'); Exit; @@ -597,6 +512,11 @@ begin absolutebegintime := Time; DataBase := TABSDatabase.Create(Self); +} + DataBase.MaxConnections := 1; + DataBase.PageSize := 8112; + DataBase.PageCountInExtent := 8; +{ DataBase.DatabaseName := 'OLDB'; DataBase.DatabaseFileName := target; DataBase.CreateDatabase; @@ -610,22 +530,30 @@ begin Query := TABSQuery.Create(Self); Query.DatabaseName := 'OLDB'; Query.SQL.Text := - 'CREATE TABLE globals ( id AUTOINC PRIMARY KEY, name STRING(128), value STRING(128) );'; + 'CREATE TABLE globals ( id AUTOINC PRIMARY KEY, name STRING(128), ' + + 'value STRING(128) );'; Query.ExecSQL; Query.SQL.Text := - 'CREATE TABLE linkmap ( id AUTOINC PRIMARY KEY, src_id INTEGER, src_link_offset INTEGER, target_id INTEGER );'; + 'CREATE TABLE linkmap ( id AUTOINC PRIMARY KEY, src_id INTEGER, ' + + 'src_link_offset INTEGER, target_id INTEGER, INDEX idsrcid src_id, ' + + 'INDEX iddestid target_id);'; Query.ExecSQL; Query.SQL.Text := - 'CREATE TABLE rawmap ( id AUTOINC PRIMARY KEY, src_id INTEGER, src_link_offset INTEGER, sep BOOLEAN, size INTEGER, data BLOB BlobCompressionMode 9 BlobBlockSize 1024 BlobCompressionAlgorithm ZLib );'; + 'CREATE TABLE rawmap ( id AUTOINC PRIMARY KEY, src_id INTEGER, ' + + 'src_link_offset INTEGER, sep BOOLEAN, size INTEGER, ' + + 'data BLOB BlobCompressionMode 9 BlobBlockSize 1024 BlobCompressionAlgorithm ZLib, ' + + 'INDEX idsrcid src_id);'; // Query.SQL.Text:='CREATE TABLE rawmap ( id AUTOINC PRIMARY KEY, src_id INTEGER, src_link_offset INTEGER, size INTEGER, data BLOB BlobCompressionAlgorithm None );'; Query.ExecSQL; Query.SQL.Text := - 'CREATE TABLE datfiles ( id INTEGER PRIMARY KEY, extension CHAR(4), name STRING(128), contenttype INTEGER, size INTEGER, data BLOB BlobCompressionMode 9 BlobBlockSize 1024 BlobCompressionAlgorithm ZLib );'; + 'CREATE TABLE datfiles ( id INTEGER PRIMARY KEY, extension CHAR(4), ' + + 'name STRING(128), contenttype INTEGER, size INTEGER, ' + + 'data BLOB BlobCompressionMode 9 BlobBlockSize 1024 BlobCompressionAlgorithm ZLib );'; // Query.SQL.Text:='CREATE TABLE datfiles ( id INTEGER PRIMARY KEY, extension CHAR(4), name STRING(128), contenttype INTEGER, size INTEGER, data BLOB BlobCompressionAlgorithm None );'; Query.ExecSQL; - Query.SQL.Text := - 'CREATE TABLE extlist ( id AUTOINC PRIMARY KEY, ext CHAR(4), ident CHAR(16) );'; - Query.ExecSQL; +// Query.SQL.Text := +// 'CREATE TABLE extlist ( id AUTOINC PRIMARY KEY, ext CHAR(4), ident CHAR(16) );'; +// Query.ExecSQL; Query.SQL.Text := 'INSERT INTO globals (name,value) VALUES ("dbversion","' + dbversion + '");'; @@ -634,8 +562,8 @@ begin for i := 0 to High(OniDataConnection.LevelInfo.Ident) do Data[i] := OniDataConnection.LevelInfo.Ident[i]; temps := CreateHexString(Data, True); - Query.SQL.Text := 'INSERT INTO globals (name,value) VALUES ("ident","' + temps + '");'; - Query.ExecSQL; +// Query.SQL.Text := 'INSERT INTO globals (name,value) VALUES ("ident","' + temps + '");'; +// Query.ExecSQL; Query.SQL.Text := 'INSERT INTO globals (name,value) VALUES ("lvl","' + IntToStr(OniDataConnection.LevelInfo.LevelNumber) + '");'; Query.ExecSQL; @@ -774,14 +702,15 @@ begin database.Free; CloseDataConnection(DataConnections[conIndex]); +} end; -procedure TForm_LevelDB.stop_convert; +procedure TForm_LevelDB.StopConvert; begin - btn_abortok.Caption := '&Close'; +{ btn_abortok.Caption := '&Close'; btn_abortok.Default := True; converting := False; lbl_estimation.Caption := 'ABORTED'; @@ -792,7 +721,7 @@ begin begin DeleteFile(loaded_filename); end; -end; +}end; @@ -815,7 +744,7 @@ end; - +{ procedure InsertDatLinkToDB(fileid: LongWord; offset: LongWord); var link: LongWord; @@ -1847,10 +1776,10 @@ begin begin end; end; +} - - +{ procedure InsertHandler(ext: String; needed: Boolean; handler: THandler); begin SetLength(ConvertHandlers, Length(ConvertHandlers) + 1); @@ -1858,9 +1787,9 @@ begin ConvertHandlers[High(ConvertHandlers)].needed := needed; ConvertHandlers[High(ConvertHandlers)].handler := handler; end; - +} begin - InsertHandler('ABNA', False, nil); +{ InsertHandler('ABNA', False, nil); // InsertHandler('AGDB',True,AGDB); InsertHandler('AGDB', False, nil); InsertHandler('AGQC', False, nil); @@ -1958,4 +1887,4 @@ begin InsertHandler('WMM_', False, nil); InsertHandler('WMMB', True, WMMB); InsertHandler('WPGE', True, WPGE); -end. +}end.