Mrwolf

.github/workflows/Build.yml

@@ -30,12 +30,12 @@ jobs:
           - os: focal
             compiler: g++-10
             cpp: 20
-            asan: on
+            asan: off
             ubsan: off
           - os: jammy
             compiler: g++
             cpp: 20
-            asan: on
+            asan: off
             ubsan: off
 
     name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}"
@@ -161,4 +161,4 @@ jobs:
       run: |
         echo 'hello world' > demo.php 
         $GITHUB_WORKSPACE/objs/bin/kphp2cpp --cxx ${{matrix.compiler}} demo.php
-        kphp_out/server -o
+        kphp_out/server -o

.github/workflows/Dockerfile.buster

@@ -22,7 +22,7 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
       libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev mysql-server libmysqlclient-dev libldap-dev libkrb5-dev \
       postgresql postgresql-server-dev-all libnuma-dev composer && \
-    pip3 install portalocker psutil requests-toolbelt pytest pytest-xdist pytest-mysql pytest-postgresql psycopg zstandard && \
+    pip3 install portalocker psutil urllib3 requests-toolbelt pytest pytest-xdist pytest-mysql pytest-postgresql psycopg zstandard && \
     rm -rf /var/lib/apt/lists/* && \
     update-alternatives --set php /usr/bin/php7.4
 

.github/workflows/Dockerfile.focal

@@ -15,10 +15,10 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
       libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev mysql-server libmysqlclient-dev libldap-dev libkrb5-dev \
       postgresql postgresql-server-dev-all libnuma-dev composer unzip && \
-    pip3 install portalocker psutil requests-toolbelt pytest pytest-xdist pytest-mysql pytest-postgresql psycopg zstandard && \
+    pip3 install portalocker psutil urllib3 requests-toolbelt pytest pytest-xdist pytest-mysql pytest-postgresql psycopg zstandard && \
     rm -rf /var/lib/apt/lists/*
 
-ENV ASAN_OPTIONS=detect_leaks=0
+ENV ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=1
 ENV UBSAN_OPTIONS=print_stacktrace=1:allow_addr2line=1
 
 RUN useradd -ms /bin/bash kitten

.github/workflows/Dockerfile.jammy

@@ -11,13 +11,18 @@ RUN apt update && \
     add-apt-repository ppa:ondrej/php -y && \
     apt update && \
     apt install -y --no-install-recommends \
-      git cmake make g++ lld gperf netcat \
+      git cmake make g++ lld gperf netcat python3 \
       python3-minimal python3-dev libpython3-dev python3-jsonschema python3-setuptools python3-pip && \
+    apt install -y --no-install-recommends \
+        php7.4-dev php7.4-common php7.4-bcmath php7.4-mbstring php7.4-curl php7.4-zstd && \
+    git clone https://github.com/uber/h3.git && cd h3 && git checkout stable-3.x && cmake -DBUILD_SHARED_LIBS=ON . && \
+    make -j4 && make install && cp /usr/local/lib/libh3.* /usr/lib/ && ldconfig && cd .. && \
+    git clone https://github.com/neatlife/php-h3.git && cd php-h3 && phpize && ./configure && make && make install && cd .. && \
     pip3 install wheel && \
     apt install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
-      libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev mysql-server libmysqlclient-dev libnuma-dev unzip \
+      libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev mysql-server libmysqlclient-dev libnuma-dev unzip \
       libldap-dev libkrb5-dev postgresql postgresql-server-dev-all && \
-    pip3 install portalocker psutil requests-toolbelt pytest pytest-xdist pytest-mysql pytest-postgresql psycopg zstandard && \
+    pip3 install portalocker psutil urllib3 requests-toolbelt pytest pytest-xdist pytest-mysql pytest-postgresql psycopg zstandard && \
     rm -rf /var/lib/apt/lists/*
 
 # set php7.4 as default
@@ -32,7 +37,7 @@ RUN update-alternatives --install /usr/bin/ld ld /usr/bin/x86_64-linux-gnu-ld 10
 RUN php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" && \
     php composer-setup.php --install-dir=/usr/bin --filename=composer --version=1.10.26
 
-ENV ASAN_OPTIONS=detect_leaks=0
+ENV ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=1
 ENV UBSAN_OPTIONS=print_stacktrace=1:allow_addr2line=1
 
 RUN useradd -ms /bin/bash kitten

common/parallel/counter-test.cpp

@@ -14,9 +14,9 @@
 
 TEST(parallel_counter, basic) {
   PARALLEL_COUNTER(counter);
-  const int nr_threads = 8;
+  const int nr_threads = 2;
   auto random_engine = std::default_random_engine();
-  std::uniform_int_distribution<int> distribution(0, 100000);
+  std::uniform_int_distribution<int> distribution(0, 1000);
 
   std::vector<std::thread> threads(nr_threads);
 

common/parallel/limit-counter-test.cpp

@@ -13,13 +13,13 @@
 #include "common/parallel/limit-counter.h"
 
 TEST(parallel_limit_counter, basic) {
-  constexpr std::size_t global_max = 100000;
-  constexpr std::size_t thread_max = 10000;
+  constexpr std::size_t global_max = 1000;
+  constexpr std::size_t thread_max = 100;
 
   PARALLEL_LIMIT_COUNTER(limit_counter);
   PARALLEL_LIMIT_COUNTER_INIT(limit_counter, global_max, thread_max);
 
-  constexpr int nr_threads = 8;
+  constexpr int nr_threads = 2;
   std::mt19937 random_engine(std::random_device{}());
   std::uniform_int_distribution<int> distribution(0, thread_max);
 

runtime/interface.cpp

@@ -1881,13 +1881,17 @@ int32_t ini_set_from_config(const char *config_file_name) {
     if (pos == std::string::npos) {
       return line_num;
     }
+    fprintf(stdout, "INI_SET: %s, %s = %s", config_file_name, 
+      string(line.substr(0, pos).data(), static_cast<string::size_type>(line.substr(0, pos).size())).c_str(),
+      string(line.substr(pos + 1).data(), static_cast<string::size_type>(line.substr(pos + 1).size())).c_str());
     ini_set(line.substr(0, pos), line.substr(pos + 1));
   }
   return 0;
 }
 
 Optional<string> f$ini_get(const string &s) {
-  if (ini_vars != nullptr && ini_vars->has_key(s)) {
+  fprintf(stdout, "INI_GET: %s, %d, %d", s.c_str(), ini_vars != nullptr, ini_vars->has_key(s));
+  if (ini_vars != nullptr && ini_vars->has_key(s)) { 
     return ini_vars->get_value(s);
   }
 

server/php-runner.cpp

@@ -78,10 +78,13 @@ void PhpScript::error(const char *error_message, script_error_t error_type) noex
   current_script->error_message = error_message;
   current_script->error_type = error_type;
   stack_end = reinterpret_cast<char *>(exit_context.uc_stack.ss_sp) + exit_context.uc_stack.ss_size;
-#if ASAN_ENABLED
-  __sanitizer_start_switch_fiber(nullptr, main_thread_stack, main_thread_stacksize);
-#endif
-  setcontext_portable(&exit_context);
+  #if ASAN_ENABLED
+    __sanitizer_start_switch_fiber(nullptr, main_thread_stack, main_thread_stacksize);
+  #endif
+    setcontext_portable(&exit_context);
+  #if ASAN_ENABLED
+    __sanitizer_finish_switch_fiber(nullptr, &main_thread_stack, &main_thread_stacksize);
+  #endif
 }
 
 void PhpScript::check_delayed_errors() noexcept {
@@ -165,12 +168,26 @@ void PhpScript::init(script_t *script, php_query_data *data_to_set) noexcept {
 
   assert_state(run_state_t::before_init);
 
+  #if ASAN_ENABLED
+    __sanitizer_start_switch_fiber(nullptr, main_thread_stack, main_thread_stacksize);
+  #endif
   getcontext_portable(&run_context);
+  #if ASAN_ENABLED
+    __sanitizer_finish_switch_fiber(nullptr, &main_thread_stack, &main_thread_stacksize);
+  #endif
+
   run_context.uc_stack.ss_sp = script_stack.get_stack_ptr();
   run_context.uc_stack.ss_size = script_stack.get_stack_size();
   run_context.uc_link = nullptr;
-  makecontext_portable(&run_context, &script_context_entrypoint, 0);
 
+  #if ASAN_ENABLED
+    __sanitizer_start_switch_fiber(nullptr, main_thread_stack, main_thread_stacksize);
+  #endif
+  makecontext_portable(&run_context, &script_context_entrypoint, 0);
+  #if ASAN_ENABLED
+    __sanitizer_finish_switch_fiber(nullptr, &main_thread_stack, &main_thread_stacksize);
+  #endif
+
   run_main = script;
   data = data_to_set;
 

tests/python/lib/http_client.py

@@ -1,6 +1,7 @@
 from requests_toolbelt.utils import dump
 import requests
 import socket
+from urllib.request import urlopen, Request
 
 from .colors import blue
 
@@ -23,15 +24,17 @@ def __init__(self, raw_bytes):
 
 
 def send_http_request(port, uri='/', method='GET', timeout=30, **kwargs):
-    session = requests.session()
+    # session = requests.session()
+    # url = 'http://127.0.0.1:{}{}'.format(port, uri)
+    # print("\nSending HTTP request: [{}]".format(blue(url)))
+    # r = session.request(method=method, url=url, timeout=timeout, **kwargs)
+    # session.close()
+    # print("HTTP request debug:")
+    # print("=============================")
+    # print(*[i for i in dump.dump_all(r).splitlines(True)], sep="\n")
+    # print("=============================")
     url = 'http://127.0.0.1:{}{}'.format(port, uri)
-    print("\nSending HTTP request: [{}]".format(blue(url)))
-    r = session.request(method=method, url=url, timeout=timeout, **kwargs)
-    session.close()
-    print("HTTP request debug:")
-    print("=============================")
-    print(*[i for i in dump.dump_all(r).splitlines(True)], sep="\n")
-    print("=============================")
+    r = urlopen(Request(method=method, url=url)).read().decode('utf-8')
     return r
 
 

tests/python/lib/stats_receiver.py

@@ -63,8 +63,10 @@ def wait_next_stats(self, timeout=60):
 
     def try_update_stats(self):
         new_stats = {}
-        for stat_line in filter(None, self._stats_file_read_fd.readlines()):
+        lines = self._stats_file_read_fd.readlines()
+        for stat_line in filter(None, lines):
             if stat_line[-1] != "\n":
+                print(lines)
                 raise RuntimeError("Got bad stat line: {}".format(stat_line))
             stat, value = stat_line.split(":")
             value, _ = value.split("|")

tests/python/tests/instance_cache/test_store_fetch_delete.py

@@ -15,7 +15,7 @@ def test_store_fetch_delete(self):
 
             resp = self.kphp_server.http_post(
                 uri="/fetch_and_verify",
-                json={"key": "key{}".format(i)})
+                json={"key": f"key{i}"})
             self.assertEqual(resp.status_code, 200)
             self.assertEqual(resp.json(), {"a": True, "b": True, "c": True})
 

tests/python/tests/job_workers/test_job_errors.py

@@ -76,8 +76,40 @@ def test_job_exception_error(self):
         ])
 
     def test_job_stack_overflow_error(self):
-        self.job_error_test_impl("stack_overflow", self.JOB_STACK_OVERFLOW_ERROR, data=[[1, 2, 3, 4, 5]], buffers=2)
-        self.kphp_server.assert_log([
+        error_code = self.JOB_STACK_OVERFLOW_ERROR
+        data = [[1, 2, 3, 4], [7, 9, 12]]
+        buffers = 4
+        stats_before = self.kphp_server.get_stats()
+        resp = self.kphp_server.http_post(
+        uri="/test_job_errors",
+        json={
+            "tag": "x2_with_error",
+            "error-type": "stack_overflow",
+            "data": data
+        })
+        self.assertEqual(resp.status_code, 200)
+
+        job_result = resp.json()["jobs-result"]
+        results = 2
+
+        eq = 0
+        got_error_code = ''
+        for i in range(results):
+            if job_result[i]["error_code"] == error_code: eq += 1
+            else: got_error_code = job_result[i]["error_code"]
+        if eq == 0: self.assertEqual(got_error_code, error_code)
+
+        corr_buffers = buffers-(results-eq)
+
+        self.kphp_server.assert_stats(
+            initial_stats=stats_before,
+            expected_added_stats={
+            "kphp_server.workers_job_memory_messages_shared_messages_buffers_acquired": corr_buffers,
+            "kphp_server.workers_job_memory_messages_shared_messages_buffers_released": corr_buffers,
+            "kphp_server.workers_job_memory_messages_shared_messages_buffer_acquire_fails": 0
+        })
+
+        self.kphp_server.assert_log(eq * [
             "Critical error during script execution: sigsegv\\(stack overflow\\)",
             "Error -1: Callstack overflow"
         ])