Skip to content

Commit c372a8b

Browse files
ziniczinic
committed
feat(cysql): switch to flat ID carry-throughs to ensure index compatibility - the pg query planner is blind to composite types in certain situations
1 parent 734be68 commit c372a8b

33 files changed

Lines changed: 541 additions & 1804 deletions

‎Makefile‎

Lines changed: 92 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,102 @@
11
THIS_FILE := $(lastword $(MAKEFILE_LIST))
22

3-
GO_CMD?=go
4-
CGO_ENABLED?=0
3+
# Go configuration
4+
GO_CMD ?= go
5+
CGO_ENABLED ?= 0
56

6-
MAIN_PACKAGES=$$($(GO_CMD) list ./...)
7+
# Main packages to test/build
8+
MAIN_PACKAGES := $(shell $(GO_CMD) list ./...)
79

8-
default: format test
9-
all: generate format test
10+
# Default target
11+
default: help
12+
all: generate format tidy lint test
1013

11-
generate:
12-
@$(GO_CMD) generate ./...
14+
# Build targets
15+
build:
16+
@echo "Building all packages..."
17+
@$(GO_CMD) build ./...
18+
19+
# Dependency management
20+
deps:
21+
@echo "Downloading dependencies..."
22+
@$(GO_CMD) mod download
23+
24+
tidy:
25+
@echo "Tidying go modules..."
26+
@$(GO_CMD) mod tidy
27+
28+
# Code quality
29+
lint:
30+
@echo "Running linter..."
31+
@$(GO_CMD) vet ./...
1332

1433
format:
34+
@echo "Formatting code..."
1535
@find ./ -name '*.go' -print0 | xargs -P 12 -0 -I '{}' goimports -w '{}'
1636

37+
# Test targets
1738
test:
18-
@for pkg in $(MAIN_PACKAGES) ; do \
19-
$(GO_CMD) test -cover $$pkg -parallel=20 ; \
20-
done
39+
@echo "Running tests..."
40+
@$(GO_CMD) test -race -cover -count=1 -parallel=10 $(MAIN_PACKAGES)
41+
42+
test_bench:
43+
@echo "Running benchmark tests..."
44+
@$(GO_CMD) test -tags benchmarks -race -cover -count=1 -parallel=1 $(MAIN_PACKAGES)
45+
46+
test_neo4j:
47+
@echo "Running Neo4j integration tests..."
48+
@$(GO_CMD) test -tags neo4j_integration -race -cover -count=1 -parallel=1 $(MAIN_PACKAGES)
49+
50+
test_pg:
51+
@echo "Running PostgreSQL integration tests..."
52+
@$(GO_CMD) test -tags pg_integration -race -cover -count=1 -parallel=1 $(MAIN_PACKAGES)
53+
54+
test_update:
55+
@echo "Updating test cases..."
56+
@CYSQL_UPDATE_CASES=true $(GO_CMD) test -parallel=10 $(MAIN_PACKAGES)
57+
58+
@cp -fv cypher/analyzer/updated_cases/* cypher/test/cases
59+
@rm -rf cypher/analyzer/updated_cases/
60+
@cp -fv cypher/models/pgsql/test/updated_cases/* cypher/models/pgsql/test/translation_cases
61+
@rm -rf cypher/analyzer/updated_cases/
62+
63+
# Utility targets
64+
generate:
65+
@echo "Running code generation..."
66+
@$(GO_CMD) generate ./...
67+
68+
clean:
69+
@echo "Cleaning build artifacts..."
70+
@$(GO_CMD) clean ./...
71+
@rm -rf cypher/analyzer/updated_cases/
72+
73+
help:
74+
@echo "Available targets:"
75+
@echo " default - Show this help message"
76+
@echo " all - Generate, format, and test (default: help)"
77+
@echo ""
78+
@echo "Build:"
79+
@echo " build - Build all packages"
80+
@echo ""
81+
@echo "Dependencies:"
82+
@echo " deps - Download dependencies"
83+
@echo " tidy - Tidy go modules"
84+
@echo ""
85+
@echo "Code Quality:"
86+
@echo " lint - Run go vet"
87+
@echo " format - Format all Go files"
88+
@echo " generate - Run code generation"
89+
@echo ""
90+
@echo "Testing:"
91+
@echo " test - Run all unit tests with coverage"
92+
@echo " test_bench - Run benchmark test"
93+
@echo " test_neo4j - Run Neo4j integration tests"
94+
@echo " test_pg - Run PostgreSQL integration tests"
95+
@echo " test_update - Update test cases"
96+
@echo ""
97+
@echo "Utility:"
98+
@echo " clean - Clean build artifacts"
99+
@echo " help - Show this help message"
100+
101+
cp -fv cypher/models/pgsql/test/updated_cases/* cypher/models/pgsql/test/translation_cases
102+
rm -rf cypher/models/pgsql/test/updated_cases

‎README.md‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,46 @@ development processes.
3333
```bash
3434
make
3535
```
36+
37+
#### Integration Tests
38+
39+
Integration tests are excluded from the default `make` target and require a running database instance. They are
40+
selected via Go build tags and configured through environment variables.
41+
42+
##### PostgreSQL Integration Tests
43+
44+
The following tests require a live PostgreSQL instance:
45+
46+
- `cypher/models/pgsql/test/translation_integration_test.go`
47+
- `cypher/models/pgsql/test/semantic_integration_test.go`
48+
49+
Set the `PG_CONNECTION_STRING` environment variable to a valid PostgreSQL connection string (e.g. `user=postgres dbname=bhe password=bhe4eva host=localhost`), then run:
50+
51+
```bash
52+
PG_CONNECTION_STRING="<connection-string>" make test_pg
53+
```
54+
55+
To run a specific test directly:
56+
57+
```bash
58+
PG_CONNECTION_STRING="<connection-string>" go test -tags pg_integration ./cypher/models/pgsql/test/...
59+
```
60+
61+
##### Neo4j Integration Tests
62+
63+
The following test requires a live Neo4j instance:
64+
65+
- `drivers/neo4j/batch_integration_test.go`
66+
67+
Set the `NEO4J_CONNECTION_STRING` environment variable to a valid Neo4j connection string (e.g.
68+
`neo4j://user:password@host:port`), then run:
69+
70+
```bash
71+
NEO4J_CONNECTION_STRING="<connection-string>" make test_neo4j
72+
```
73+
74+
To run the batch integration test directly:
75+
76+
```bash
77+
NEO4J_CONNECTION_STRING="<connection-string>" go test -tags neo4j_integration ./drivers/neo4j/...
78+
```

‎cardinality/hyperloglog32_test.go‎

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -6,37 +6,6 @@ import (
66
"github.com/stretchr/testify/require"
77
)
88

9-
// This is a test that serves as a documented example of how HLL works. This test was designed to use the 14 register
10-
// HLL implementation.
11-
//
12-
// For more information on HLL see: https://en.wikipedia.org/wiki/HyperLogLog
13-
func TestHyperLogLog32(t *testing.T) {
14-
const cardinalityMax = 10_000_000
15-
16-
sketch := NewHyperLogLog32()
17-
18-
for i := uint32(0); i < cardinalityMax; i++ {
19-
sketch.Add(i)
20-
}
21-
22-
var (
23-
estimatedCardinality = sketch.Cardinality()
24-
deviation = 100 - cardinalityMax/float64(estimatedCardinality)*100
25-
)
26-
27-
// We expect the HLL sketch to have a cardinality that does not deviate more than 0.5% from reality
28-
require.Truef(t, deviation < 0.5, "Expected a cardinality less than 0.5%% but got %.2f%%", deviation)
29-
30-
for i := 0; i < 100; i++ {
31-
previous := sketch.Cardinality()
32-
33-
sketch.Add(0)
34-
after := sketch.Cardinality()
35-
36-
require.Equal(t, previous, after, "Expected cardinality to remain the same after encoding the same ID twice")
37-
}
38-
}
39-
409
func TestHyperLogLog32_Add(t *testing.T) {
4110
sketch := NewHyperLogLog32()
4211

‎cardinality/hyperloglog64_test.go‎

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -6,37 +6,6 @@ import (
66
"github.com/stretchr/testify/require"
77
)
88

9-
// This is a test that serves as a documented example of how HLL works. This test was designed to use the 14 register
10-
// HLL implementation.
11-
//
12-
// For more information on HLL see: https://en.wikipedia.org/wiki/HyperLogLog
13-
func TestHyperLogLog64(t *testing.T) {
14-
const cardinalityMax = 10_000_000
15-
16-
sketch := NewHyperLogLog64()
17-
18-
for i := uint64(0); i < cardinalityMax; i++ {
19-
sketch.Add(i)
20-
}
21-
22-
var (
23-
estimatedCardinality = sketch.Cardinality()
24-
deviation = 100 - cardinalityMax/float64(estimatedCardinality)*100
25-
)
26-
27-
// We expect the HLL sketch to have a cardinality that does not deviate more than 0.66% from reality
28-
require.Truef(t, deviation < 0.66, "Expected a cardinality less than 0.66%% but got %.2f%%", deviation)
29-
30-
for i := 0; i < 100; i++ {
31-
previous := sketch.Cardinality()
32-
33-
sketch.Add(0)
34-
after := sketch.Cardinality()
35-
36-
require.Equal(t, previous, after, "Expected cardinality to remain the same after encoding the same ID twice")
37-
}
38-
}
39-
409
func TestHyperLogLog64_Add(t *testing.T) {
4110
sketch := NewHyperLogLog64()
4211

‎cardinality/hyperloglog_bench_test.go‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
//go:build benchmarks
2+
13
package cardinality_test
24

35
import (
@@ -6,6 +8,7 @@ import (
68
"testing"
79

810
"github.com/axiomhq/hyperloglog"
11+
"github.com/stretchr/testify/require"
912
)
1013

1114
var (
@@ -345,3 +348,65 @@ func BenchmarkAddUint32_NoPool_10000000(b *testing.B) {
345348
sketch.AddUint32NoPool(uint32Slice...)
346349
}
347350
}
351+
352+
// This is a test that serves as a documented example of how HLL works. This test was designed to use the 14 register
353+
// HLL implementation.
354+
//
355+
// For more information on HLL see: https://en.wikipedia.org/wiki/HyperLogLog
356+
func TestHyperLogLog32(t *testing.T) {
357+
const cardinalityMax = 10_000_000
358+
359+
sketch := NewHyperLogLog32()
360+
361+
for i := uint32(0); i < cardinalityMax; i++ {
362+
sketch.Add(i)
363+
}
364+
365+
var (
366+
estimatedCardinality = sketch.Cardinality()
367+
deviation = 100 - cardinalityMax/float64(estimatedCardinality)*100
368+
)
369+
370+
// We expect the HLL sketch to have a cardinality that does not deviate more than 0.5% from reality
371+
require.Truef(t, deviation < 0.5, "Expected a cardinality less than 0.5%% but got %.2f%%", deviation)
372+
373+
for i := 0; i < 100; i++ {
374+
previous := sketch.Cardinality()
375+
376+
sketch.Add(0)
377+
after := sketch.Cardinality()
378+
379+
require.Equal(t, previous, after, "Expected cardinality to remain the same after encoding the same ID twice")
380+
}
381+
}
382+
383+
// This is a test that serves as a documented example of how HLL works. This test was designed to use the 14 register
384+
// HLL implementation.
385+
//
386+
// For more information on HLL see: https://en.wikipedia.org/wiki/HyperLogLog
387+
func TestHyperLogLog64(t *testing.T) {
388+
const cardinalityMax = 10_000_000
389+
390+
sketch := NewHyperLogLog64()
391+
392+
for i := uint64(0); i < cardinalityMax; i++ {
393+
sketch.Add(i)
394+
}
395+
396+
var (
397+
estimatedCardinality = sketch.Cardinality()
398+
deviation = 100 - cardinalityMax/float64(estimatedCardinality)*100
399+
)
400+
401+
// We expect the HLL sketch to have a cardinality that does not deviate more than 0.66% from reality
402+
require.Truef(t, deviation < 0.66, "Expected a cardinality less than 0.66%% but got %.2f%%", deviation)
403+
404+
for i := 0; i < 100; i++ {
405+
previous := sketch.Cardinality()
406+
407+
sketch.Add(0)
408+
after := sketch.Cardinality()
409+
410+
require.Equal(t, previous, after, "Expected cardinality to remain the same after encoding the same ID twice")
411+
}
412+
}

‎cypher/models/pgsql/test/fixture.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ import (
2222
"github.com/specterops/dawgs/graph"
2323
)
2424

25+
const (
26+
PGConnectionStringEV = "PG_CONNECTION_STRING"
27+
)
28+
2529
var (
2630
// Node and edge kinds to keep queries consistent
2731
NodeKind1 = graph.StringKind("NodeKind1")

‎cypher/models/pgsql/test/semantic_integration_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
//go:build manual_integration
1+
//go:build pg_integration
22

33
// Copyright 2026 Specter Ops, Inc.
44
//

‎cypher/models/pgsql/test/translation_cases/delete.sql‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,11 @@
1515
-- SPDX-License-Identifier: Apache-2.0
1616

1717
-- case: match (s:NodeKind1) detach delete s
18-
with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (delete from node n1 using s0 where (s0.n0).id = n1.id) select 1;
18+
with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n0.id as n0_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (delete from node n1 using s0 where s0.n0_id = n1.id) select 1;
1919

2020
-- case: match ()-[r:EdgeKind1]->() delete r
21-
with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1;
21+
with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n0.id as n0_id, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, n1.id as n1_id from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1;
2222

2323
-- case: match ()-[]->()-[r:EdgeKind1]->() delete r
24-
with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.kind_id = any (array [3]::int2[])), s2 as (delete from edge e2 using s1 where (s1.e1).id = e2.id) select 1;
24+
with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n0.id as n0_id, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, n1.id as n1_id from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s0.n0 as n0, s0.n0_id as n0_id, s0.n1 as n1, s0.n1_id as n1_id, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, n2.id as n2_id from s0 join edge e1 on s0.n1_id = e1.start_id join node n2 on n2.id = e1.end_id where e1.kind_id = any (array [3]::int2[])), s2 as (delete from edge e2 using s1 where (s1.e1).id = e2.id) select 1;
2525

0 commit comments

Comments
 (0)