Skip to content

Commit 52d96b6

Browse files
FrottyFrotty
committed
some caching
1 parent a8bfb86 commit 52d96b6

7 files changed

Lines changed: 855 additions & 287 deletions

File tree

‎de.peeeq.wurstscript/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ dependencies {
9898
implementation 'com.github.albfernandez:juniversalchardet:2.4.0'
9999
implementation 'com.github.inwc3:jmpq3:1676fc7020'
100100
implementation 'com.github.inwc3:wc3libs:e33d3dc368'
101-
implementation 'com.github.wurstscript:wurstsetup:475cc7fae8'
101+
implementation 'com.github.wurstscript:wurstsetup:393cf5ea39'
102102
implementation 'org.slf4j:slf4j-api:1.7.25'
103103
implementation 'ch.qos.logback:logback-classic:1.5.13'
104104
implementation 'org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r'

‎de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/intermediateLang/interpreter/ProgramStateIO.java‎

Lines changed: 187 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package de.peeeq.wurstio.intermediateLang.interpreter;
22

33
import com.google.common.collect.Maps;
4+
import de.peeeq.wurstio.map.importer.ImportFile;
45
import de.peeeq.wurstio.mpq.MpqEditor;
56
import de.peeeq.wurstio.objectreader.ObjectFileType;
67
import de.peeeq.wurstscript.WLogger;
@@ -23,23 +24,106 @@
2324
import java.nio.charset.StandardCharsets;
2425
import java.nio.file.Files;
2526
import java.nio.file.Path;
26-
import java.util.List;
27-
import java.util.Map;
28-
import java.util.Optional;
27+
import java.util.*;
28+
import java.util.stream.Collectors;
2929

3030
public class ProgramStateIO extends ProgramState {
3131

3232
private static final int GENERATED_BY_WURST = 42;
33+
private static final String OBJECT_CACHE_MANIFEST = "wurst_object_cache.txt";
34+
3335
private @Nullable ImStmt lastStatement;
34-
private @Nullable
35-
final MpqEditor mpqEditor;
36+
private @Nullable final MpqEditor mpqEditor;
3637
private final Map<ObjectFileType, ObjMod<? extends ObjMod.Obj>> dataStoreMap = Maps.newLinkedHashMap();
38+
private final Map<ObjectFileType, String> dataStoreHashes = Maps.newLinkedHashMap();
3739
private int id = 0;
3840
private final Map<String, ObjMod.Obj> objDefinitions = Maps.newLinkedHashMap();
3941
private PrintStream outStream = System.err;
4042
private @Nullable WTS trigStrings = null;
4143
private final Optional<File> mapFile;
4244

45+
/**
46+
* Tracks which object files have been modified during compiletime
47+
*/
48+
private static class ObjectCacheManifest {
49+
Map<String, ObjectFileEntry> objectFiles = new HashMap<>();
50+
51+
static class ObjectFileEntry {
52+
String fileType;
53+
String hash;
54+
long timestamp;
55+
int objectCount;
56+
57+
ObjectFileEntry(String fileType, String hash, long timestamp, int objectCount) {
58+
this.fileType = fileType;
59+
this.hash = hash;
60+
this.timestamp = timestamp;
61+
this.objectCount = objectCount;
62+
}
63+
}
64+
65+
String serialize() {
66+
StringBuilder sb = new StringBuilder();
67+
sb.append("# Wurst Object Cache Manifest v1\n");
68+
sb.append("# Format: fileType|hash|timestamp|objectCount\n");
69+
70+
for (Map.Entry<String, ObjectFileEntry> entry : objectFiles.entrySet()) {
71+
ObjectFileEntry e = entry.getValue();
72+
sb.append(e.fileType)
73+
.append("|")
74+
.append(e.hash)
75+
.append("|")
76+
.append(e.timestamp)
77+
.append("|")
78+
.append(e.objectCount)
79+
.append("\n");
80+
}
81+
return sb.toString();
82+
}
83+
84+
static ObjectCacheManifest deserialize(String data) {
85+
ObjectCacheManifest manifest = new ObjectCacheManifest();
86+
if (data == null || data.isEmpty()) {
87+
return manifest;
88+
}
89+
90+
String[] lines = data.split("\n");
91+
for (String line : lines) {
92+
if (line.startsWith("#") || line.trim().isEmpty()) {
93+
continue;
94+
}
95+
String[] parts = line.split("\\|");
96+
if (parts.length == 4) {
97+
try {
98+
String fileType = parts[0];
99+
String hash = parts[1];
100+
long timestamp = Long.parseLong(parts[2]);
101+
int objectCount = Integer.parseInt(parts[3]);
102+
103+
manifest.objectFiles.put(fileType,
104+
new ObjectFileEntry(fileType, hash, timestamp, objectCount));
105+
} catch (NumberFormatException e) {
106+
WLogger.warning("Invalid object cache manifest line: " + line);
107+
}
108+
}
109+
}
110+
return manifest;
111+
}
112+
113+
boolean hasEntry(String fileType) {
114+
return objectFiles.containsKey(fileType);
115+
}
116+
117+
boolean hashMatches(String fileType, String hash) {
118+
ObjectFileEntry entry = objectFiles.get(fileType);
119+
return entry != null && entry.hash.equals(hash);
120+
}
121+
122+
void putEntry(String fileType, String hash, int objectCount) {
123+
objectFiles.put(fileType, new ObjectFileEntry(fileType, hash, System.currentTimeMillis(), objectCount));
124+
}
125+
}
126+
43127
public ProgramStateIO(Optional<File> mapFile, @Nullable MpqEditor mpqEditor, WurstGui gui, ImProg prog, boolean isCompiletime) {
44128
super(gui, prog, isCompiletime);
45129
this.mapFile = mapFile;
@@ -87,7 +171,6 @@ ObjMod<? extends ObjMod.Obj> getDataStore(String fileExtension) {
87171
return getDataStore(ObjectFileType.fromExt(fileExtension));
88172
}
89173

90-
91174
private ObjMod<? extends ObjMod.Obj> getDataStore(ObjectFileType filetype) throws Error {
92175
ObjMod<? extends ObjMod.Obj> dataStore = dataStoreMap.get(filetype);
93176
if (dataStore != null) {
@@ -274,7 +357,6 @@ private void deleteWurstObjects(ObjMod<? extends ObjMod.Obj> dataStore) {
274357
}
275358
}
276359

277-
278360
String addObjectDefinition(ObjMod.Obj objDef) {
279361
id++;
280362
String key = "obj" + id;
@@ -286,21 +368,114 @@ ObjMod.Obj getObjectDefinition(String key) {
286368
return objDefinitions.get(key);
287369
}
288370

371+
/**
372+
* Calculate hash of an object file's contents
373+
*/
374+
private String calculateObjectFileHash(ObjMod<? extends ObjMod.Obj> dataStore) {
375+
try {
376+
ByteArrayOutputStream baos = new ByteArrayOutputStream();
377+
Wc3BinOutputStream out = new Wc3BinOutputStream(baos);
378+
dataStore.write(out, ObjMod.EncodingFormat.OBJ_0x2);
379+
out.close();
380+
byte[] data = baos.toByteArray();
381+
return ImportFile.calculateHash(data);
382+
} catch (Exception e) {
383+
WLogger.warning("Could not calculate object file hash: " + e.getMessage());
384+
return String.valueOf(System.currentTimeMillis());
385+
}
386+
}
387+
388+
/**
389+
* Load the object cache manifest from MPQ
390+
*/
391+
private ObjectCacheManifest loadObjectCacheManifest() {
392+
if (mpqEditor == null) {
393+
return new ObjectCacheManifest();
394+
}
395+
396+
try {
397+
if (mpqEditor.hasFile(OBJECT_CACHE_MANIFEST)) {
398+
byte[] data = mpqEditor.extractFile(OBJECT_CACHE_MANIFEST);
399+
String content = new String(data, StandardCharsets.UTF_8);
400+
WLogger.info("Loaded object cache manifest from MPQ");
401+
return ObjectCacheManifest.deserialize(content);
402+
}
403+
} catch (Exception e) {
404+
WLogger.info("Could not load object cache manifest: " + e.getMessage());
405+
}
406+
return new ObjectCacheManifest();
407+
}
408+
409+
/**
410+
* Save the object cache manifest to MPQ
411+
*/
412+
private void saveObjectCacheManifest(ObjectCacheManifest manifest) {
413+
if (mpqEditor == null) {
414+
return;
415+
}
416+
417+
try {
418+
String serialized = manifest.serialize();
419+
byte[] data = serialized.getBytes(StandardCharsets.UTF_8);
420+
mpqEditor.deleteFile(OBJECT_CACHE_MANIFEST);
421+
mpqEditor.insertFile(OBJECT_CACHE_MANIFEST, data);
422+
WLogger.info("Saved object cache manifest to MPQ");
423+
} catch (Exception e) {
424+
WLogger.warning("Could not save object cache manifest: " + e.getMessage());
425+
}
426+
}
427+
289428
@Override
290429
public void writeBack(boolean inject) {
291430
gui.sendProgress("Writing back generated objects");
431+
long startTime = System.currentTimeMillis();
432+
433+
// Load the existing cache manifest
434+
ObjectCacheManifest oldManifest = loadObjectCacheManifest();
435+
ObjectCacheManifest newManifest = new ObjectCacheManifest();
436+
437+
int filesProcessed = 0;
438+
int filesUpdated = 0;
439+
int filesSkipped = 0;
292440

293441
for (ObjectFileType fileType : ObjectFileType.values()) {
294-
WLogger.info("Writing back " + fileType);
442+
filesProcessed++;
295443
ObjMod<? extends ObjMod.Obj> dataStore = getDataStore(fileType);
444+
296445
if (!dataStore.getObjsList().isEmpty()) {
297-
WLogger.info("Writing back filetype " + fileType);
298-
writebackObjectFile(dataStore, fileType, inject);
446+
// Calculate hash of current object file
447+
String currentHash = calculateObjectFileHash(dataStore);
448+
dataStoreHashes.put(fileType, currentHash);
449+
450+
// Check if it matches the cached version
451+
if (oldManifest.hasEntry(fileType.getExt()) &&
452+
oldManifest.hashMatches(fileType.getExt(), currentHash)) {
453+
454+
WLogger.info("Object file " + fileType.getExt() + " unchanged (hash match), skipping writeback");
455+
filesSkipped++;
456+
457+
// Still add to new manifest
458+
newManifest.putEntry(fileType.getExt(), currentHash, dataStore.getObjsList().size());
459+
} else {
460+
WLogger.info("Object file " + fileType.getExt() + " changed or new, writing back");
461+
filesUpdated++;
462+
writebackObjectFile(dataStore, fileType, inject);
463+
newManifest.putEntry(fileType.getExt(), currentHash, dataStore.getObjsList().size());
464+
}
299465
} else {
300-
WLogger.info("Writing back empty for " + fileType);
466+
WLogger.info("Object file " + fileType.getExt() + " is empty, skipping");
301467
}
302468
}
469+
470+
// Always write w3o file (it's relatively cheap)
303471
writeW3oFile();
472+
473+
// Save the new manifest
474+
saveObjectCacheManifest(newManifest);
475+
476+
long endTime = System.currentTimeMillis();
477+
WLogger.info(String.format("Object writeback complete in %dms: %d files processed, %d updated, %d skipped (cached)",
478+
endTime - startTime, filesProcessed, filesUpdated, filesSkipped));
304479
}
305480

306481
private void writeW3oFile() {
@@ -322,7 +497,6 @@ private void writeW3oFile() {
322497
}
323498

324499
private void writebackObjectFile(ObjMod<? extends ObjMod.Obj> dataStore, ObjectFileType fileType, boolean inject) throws Error {
325-
326500
try {
327501
ByteArrayOutputStream baos = new ByteArrayOutputStream();
328502
Wc3BinOutputStream out = new Wc3BinOutputStream(baos);
@@ -347,7 +521,6 @@ private void writebackObjectFile(ObjMod<? extends ObjMod.Obj> dataStore, ObjectF
347521
WLogger.severe(e);
348522
throw new Error(e);
349523
}
350-
351524
}
352525

353526
public void exportToWurst(ObjMod<? extends ObjMod.Obj> dataStore,
@@ -366,14 +539,13 @@ public void exportToWurst(ObjMod<? extends ObjMod.Obj> dataStore,
366539
}
367540
}
368541

369-
370542
public void exportToWurst(List<? extends ObjMod.Obj> customObjs, ObjectFileType fileType, Appendable out) throws IOException {
371543
for (ObjMod.Obj obj : customObjs) {
372544
String oldId = obj.getBaseId().getVal();
373545
String newId = (obj.getNewId() != null ? obj.getNewId().getVal() : "xxxx");
374546
out.append("@compiletime function create_").append(fileType.getExt()).append("_").append(newId)
375547
.append("()\n");
376-
out.append(" let def = createObjectDefinition(\"").append(fileType.getExt()).append("\", '");
548+
out.append("\tlet def = createObjectDefinition(\"").append(fileType.getExt()).append("\", '");
377549
out.append(newId);
378550
out.append("', '");
379551
out.append(oldId);
@@ -417,7 +589,6 @@ private String valTypeToFuncPostfix(ObjMod.ValType valType) {
417589
return "Int";
418590
}
419591

420-
421592
private Optional<File> getObjectEditingOutputFolder() {
422593
if (!mapFile.isPresent()) {
423594
File folder = new File("_build", "objectEditingOutput");
@@ -441,7 +612,4 @@ public PrintStream getOutStream() {
441612
public void setOutStream(PrintStream os) {
442613
outStream = os;
443614
}
444-
445-
446615
}
447-

0 commit comments

Comments
 (0)