FEAT: implement HAR converter with request/response handling, proxy support, and devcontainer improvements

.devcontainer/devcontainer.json

@@ -1,9 +1,10 @@
 {
     "name": "Go",
-    "dockerFile": "Dockerfile",
+    "image": "mcr.microsoft.com/devcontainers/go:1.24", // Switch to 1.25 when it's released
     "remoteUser": "vscode",
     "shutdownAction": "stopContainer",
-    "postCreateCommand": "go mod download",
+    "initializeCommand": "./.devcontainer/initialize.sh",
+    "postCreateCommand": "./.devcontainer/post-create.sh",
     "customizations": {
         "vscode": {
             "settings": {
@@ -17,5 +18,12 @@
     },
     "mounts": [
         "source=go-modules,target=/go,type=volume" // Keep go modules in a volume
-    ]
+    ],
+    "features": {
+        "ghcr.io/devcontainers-extra/features/zsh-plugins:0": {
+            "plugins": "git golang zsh-autosuggestions zsh-syntax-highlighting zsh-you-should-use",
+            "omzPlugins": "https://github.com/zsh-users/zsh-autosuggestions https://github.com/zsh-users/zsh-syntax-highlighting https://github.com/MichaelAquilina/zsh-you-should-use",
+            "username": "vscode"
+        }
+    }
 }

.devcontainer/initialize.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+if [ "$(uname)" == "Darwin" ]; then
+    CHARLES_APP=$(mdfind "kMDItemCFBundleIdentifier == 'com.xk72.Charles'" | head -n 1)
+
+    if [ -d "$CHARLES_APP" ]; then
+        rm -rf .certs
+        mkdir -p .certs
+        "$CHARLES_APP/Contents/MacOS/Charles" ssl export .certs/charles-ssl.pem
+    else
+        echo "Charles is not installed, skipping certificate export."
+    fi
+fi

.devcontainer/post-create.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+go mod download
+
+sudo cp .certs/charles-ssl.pem /usr/local/share/ca-certificates/charles-ssl-proxying-certificate.crt
+sudo update-ca-certificates

.gitignore

@@ -0,0 +1,9 @@
+# DEVCONTAINER
+.certs
+
+# DEBUG
+example-harkit
+*.har
+
+# OS
+.DS_Store

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Debug example/main.go",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${workspaceFolder}/example/main.go"
+        }
+    ]
+}

.vscode/settings.json

@@ -4,6 +4,9 @@
     "files.autoSave": "onFocusChange",
     "files.insertFinalNewline": true,
     "files.trimFinalNewlines": true,
+    "files.exclude": {
+        "**/.certs": true
+    },
     // GOLANG SETTINGS:
     "go.toolsManagement.autoUpdate": true,
     "go.useLanguageServer": true,

README.md

@@ -1,3 +1,34 @@
 # HAR file management library
 
-A Golang library for parsing and managing HAR (HTTP Archive) files. Provides easy-to-use structs and functions for loading, inspecting, and manipulating HAR files, making HTTP traffic analysis and debugging simpler.
+A Golang library for parsing and managing HAR (HTTP Archive) files. Provides easy-to-use structs and functions for loading, inspecting, and manipulating HAR files, making HTTP traffic analysis and debugging simpler. This library is designed to be used with the following libraries:
+
+* [`bogdanfin/tls-client`](https://github.com/bogdanfinn/tls-client)
+* The standard **`net/http`** library
+* Other **custom request/response structures**
+
+## Purpose
+
+* Provide a **complete and persistent history of requests and responses**.
+* Facilitate **monitoring, tracking, and debugging** of request systems.
+* Offer a **1:1 equivalent of HAR exports** produced by tools like **Charles Proxy** or **Proxyman** during SSL proxying.
+
+## Functional Requirements / Specifications
+
+* **Multi-source compatibility:** Capture and generate HAR files from **tls-client**, **net/http**, or any **custom structs**.
+* **Maximum fidelity to the HAR standard:** Match as closely as possible the format and content produced by **Charles Proxy** (as a reference for export quality).
+* **Strict header order preservation:** Maintain the exact order of headers as defined by the TLS layer (which Go does not guarantee by default—requires specific handling).
+* **Simplified API:**
+  * Create a **HAR session** via a dedicated function.
+  * Add a **request** to the HAR.
+  * Add the **corresponding response** via a complementary function.
+  * Explicit handling of **requests without responses** (timeouts, cancellations, network errors).
+* **Additional fields beyond the HAR standard:**
+  * **IP address** used during the TLS connection.
+  * **Session ID** for session monitoring and tracking.
+
+## Bonus / Potential Extensions
+
+* **Monitoring integration:** Native export compatible with **Prometheus / Grafana**, or extract metrics directly from HAR files for real-time visualization.
+* **Advanced historization:** Automatic HAR file storage in an **S3 bucket**, with associated **metadata/tags**:
+  * Final request status: **success**, **failure**, **timeout**, **HTTP 5xx**, etc.
+  * **Category / service / user tagging** for easier identification.

converter/common.go

@@ -0,0 +1,79 @@
+package converter
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/Mathious6/harkit/harfile"
+	http "github.com/bogdanfinn/fhttp"
+)
+
+const (
+	DefaultRequestHTTPVersion = "HTTP/2.0"
+	ContentLengthKey          = "Content-Length"
+	ContentTypeKey            = "Content-Type"
+	CookieKey                 = "Cookie"
+	SetCookieKey              = "Set-Cookie"
+	LocationKey               = "Location"
+)
+
+func convertCookies(cookies []*http.Cookie) []*harfile.Cookie {
+	harCookies := make([]*harfile.Cookie, len(cookies))
+	for index, cookie := range cookies {
+		harCookies[index] = &harfile.Cookie{
+			Name:     cookie.Name,
+			Value:    cookie.Value,
+			Path:     cookie.Path,
+			Domain:   cookie.Domain,
+			Expires:  formatExpires(cookie.Expires),
+			HTTPOnly: cookie.HttpOnly,
+			Secure:   cookie.Secure,
+		}
+	}
+	return harCookies
+}
+
+func formatExpires(expires time.Time) string {
+	if expires.IsZero() {
+		return ""
+	}
+	return expires.Format(time.RFC3339Nano)
+}
+
+func convertHeaders(header http.Header, contentLength int64) []*harfile.NVPair {
+	// By default, client adds Content-Length header later on, so we need to add it here.
+	// We clone the header to avoid modifying the original one to avoid side effects.
+	clonedHeader := header.Clone()
+	if contentLength > 0 && clonedHeader.Get(ContentLengthKey) == "" {
+		clonedHeader.Set(ContentLengthKey, fmt.Sprintf("%d", contentLength))
+	}
+
+	harHeaders := make([]*harfile.NVPair, 0, len(clonedHeader))
+	seen := make(map[string]bool)
+
+	// Used to sort headers in HAR file if needed (e.g. https://github.com/bogdanfinn/tls-client)
+	order := clonedHeader.Values(http.HeaderOrderKey)
+	for _, name := range order {
+		canonical := http.CanonicalHeaderKey(name)
+		values := clonedHeader.Values(name)
+
+		if len(values) > 0 {
+			for _, value := range values {
+				harHeaders = append(harHeaders, &harfile.NVPair{Name: canonical, Value: value})
+			}
+			seen[canonical] = true
+		}
+	}
+
+	for name, values := range clonedHeader {
+		if seen[name] || strings.EqualFold(name, http.HeaderOrderKey) {
+			continue
+		}
+		for _, value := range values {
+			harHeaders = append(harHeaders, &harfile.NVPair{Name: name, Value: value})
+		}
+	}
+
+	return harHeaders
+}

converter/request.go

@@ -0,0 +1,153 @@
+package converter
+
+import (
+	"errors"
+	"io"
+	"net/url"
+	"strings"
+
+	"github.com/Mathious6/harkit/harfile"
+	http "github.com/bogdanfinn/fhttp"
+)
+
+const (
+	applicationXWWWFormURLEncoded = "application/x-www-form-urlencoded"
+	multipartFormData             = "multipart/form-data"
+	maxMultipartFormDataSize      = 32 << 20 // 32 MB limit
+
+	methodKey    = ":method"
+	authorityKey = ":authority"
+	schemeKey    = ":scheme"
+	pathKey      = ":path"
+
+	hostKey = "Host"
+)
+
+func FromHTTPRequest(req *http.Request) (*harfile.Request, error) {
+	if req == nil {
+		return nil, errors.New("request cannot be nil")
+	}
+
+	reqProto := DefaultRequestHTTPVersion // WARNING: req.Proto is not always accurate so we force it.
+
+	protocolHeader := handleRequestProtocolHeader(reqProto, req.Method, *req.URL)
+	headers := convertHeaders(req.Header, req.ContentLength)
+
+	postData, err := extractRequestPostData(req)
+	if err != nil {
+		return nil, err
+	}
+
+	return &harfile.Request{
+		Method:      req.Method,
+		URL:         req.URL.String(),
+		HTTPVersion: reqProto,
+		Cookies:     convertCookies(req.Cookies()),
+		Headers:     append(protocolHeader, headers...),
+		QueryString: convertRequestQueryParams(req.URL),
+		PostData:    postData,
+		HeadersSize: -1,
+		BodySize:    req.ContentLength,
+	}, nil
+}
+
+func handleRequestProtocolHeader(proto string, method string, url url.URL) []*harfile.NVPair {
+	if proto == "HTTP/2.0" {
+		return []*harfile.NVPair{
+			{Name: methodKey, Value: method},
+			{Name: authorityKey, Value: url.Host},
+			{Name: schemeKey, Value: url.Scheme},
+			{Name: pathKey, Value: url.RequestURI()},
+		}
+	} else {
+		return []*harfile.NVPair{
+			{Name: hostKey, Value: url.Host},
+		}
+	}
+}
+
+func convertRequestQueryParams(u *url.URL) []*harfile.NVPair {
+	result := make([]*harfile.NVPair, 0)
+
+	for key, values := range u.Query() {
+		for _, value := range values {
+			result = append(result, &harfile.NVPair{Name: key, Value: value})
+		}
+	}
+
+	return result
+}
+
+func extractRequestPostData(req *http.Request) (*harfile.PostData, error) {
+	if req.Body == nil || req.ContentLength == 0 {
+		return nil, nil
+	}
+
+	body, err := req.GetBody()
+	if err != nil {
+		return nil, err
+	}
+	defer body.Close()
+
+	bodyText, err := io.ReadAll(body)
+	if err != nil {
+		return nil, err
+	}
+
+	mimeType := req.Header.Get(ContentTypeKey)
+	postData := &harfile.PostData{MimeType: mimeType}
+
+	if strings.HasPrefix(mimeType, applicationXWWWFormURLEncoded) {
+		pairs := strings.SplitSeq(string(bodyText), "&")
+
+		for pair := range pairs {
+			nv := strings.SplitN(pair, "=", 2)
+			if len(nv) == 2 {
+				name, value := nv[0], nv[1]
+				postData.Params = append(postData.Params, &harfile.Param{Name: name, Value: value})
+			}
+		}
+
+		return postData, nil
+	}
+
+	if strings.HasPrefix(mimeType, multipartFormData) {
+		err := req.ParseMultipartForm(maxMultipartFormDataSize)
+		if err != nil {
+			return nil, err
+		}
+
+		for name, values := range req.MultipartForm.Value {
+			for _, value := range values {
+				postData.Params = append(postData.Params, &harfile.Param{Name: name, Value: value})
+			}
+		}
+
+		for name, files := range req.MultipartForm.File {
+			for _, fileHeader := range files {
+				file, err := fileHeader.Open()
+				if err != nil {
+					return nil, err
+				}
+				defer file.Close()
+
+				content, err := io.ReadAll(file)
+				if err != nil {
+					return nil, err
+				}
+
+				postData.Params = append(postData.Params, &harfile.Param{
+					Name:        name,
+					FileName:    fileHeader.Filename,
+					ContentType: fileHeader.Header.Get(ContentTypeKey),
+					Value:       string(content),
+				})
+			}
+		}
+
+		return postData, nil
+	}
+
+	postData.Text = string(bodyText)
+	return postData, nil
+}