Merge remote-tracking branch 'origin/master' into r.11

# Conflicts:
#	go.mod
#	go.sum
#	pkg/drivers/cdp/helpers.go
#	pkg/drivers/http/driver.go
#	pkg/drivers/http/driver_test.go
#	pkg/drivers/http/options.go
#	pkg/drivers/http/options_test.go
#	pkg/stdlib/types/to_binary.go
#	pkg/stdlib/types/to_binary_test.go

.github/workflows/build.yml

@@ -3,27 +3,54 @@ name: build
 on: [push, pull_request]
 
 jobs:
+  analyze:
+    name: Static Analysis
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '>=1.19'
+
+      - name: Set up linters
+        run: make install-tools
+
+      - name: Lint
+        run: |
+          make vet
+          make lint
+          make fmt
+          git diff
+          if [[ $(git diff) != '' ]]; then echo 'Invalid formatting!' >&2; exit 1; fi
+
   build:
     name: Build
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        goVer: [1.14, 1.15, 1.16, 1.17]
+        goVer: [1.18, 1.19]
     steps:
       - name: Set up Go ${{ matrix.goVer }}
         uses: actions/setup-go@v2
         with:
           go-version: ${{ matrix.goVer }}
         id: go
 
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
       - name: Set up ANTLR
         env:
-          ANTLR_VERSION: 4.9
+          ANTLR_VERSION: 4.11.1
         run: |
           sudo curl -o /usr/local/lib/antlr-${ANTLR_VERSION}-complete.jar https://www.antlr.org/download/antlr-${ANTLR_VERSION}-complete.jar
           export CLASSPATH=".:/usr/local/lib/antlr-${ANTLR_VERSION}-complete.jar:$CLASSPATH"
           mkdir $HOME/antlr-bin
-          echo -e '#!/bin/bash\njava -jar /usr/local/lib/antlr-4.9-complete.jar "$@"' > $HOME/antlr-bin/antlr
+          echo -e '#!/bin/bash\njava -jar /usr/local/lib/antlr-4.11.1-complete.jar "$@"' > $HOME/antlr-bin/antlr
           echo -e '#!/bin/bash\njava org.antlr.v4.gui.TestRig "$@"' > $HOME/antlr-bin/grun
           chmod +x $HOME/antlr-bin/*
           export PATH=$PATH:$HOME/antlr-bin
@@ -32,25 +59,12 @@ jobs:
       - name: Set up Lab
         run: |
           curl https://raw.githubusercontent.com/MontFerret/lab/master/install.sh -o install.sh
-          sudo sh install.sh
-
-      - name: Set up linters
-        run: go get -u github.com/mgechev/revive
-
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+          chmod +x ./install.sh
+          LOCATION=$PWD ./install.sh
 
       - name: Get dependencies
         run: make install
 
-      - name: Lint
-        run: |
-          make vet
-          make lint
-          make fmt
-          git diff
-          if [[ $(git diff) != '' ]]; then echo 'Invalid formatting!' >&2; exit 1; fi
-
       - name: Generate
         run: |
           export PATH=$PATH:$HOME/antlr-bin
@@ -65,5 +79,5 @@ jobs:
       - name: E2E tests
         run: |
           docker run -d -p 9222:9222 ghcr.io/montferret/chromium:92.0.4512.0
-          make e2e
+          LAB_BIN=$PWD/lab make e2e
           docker stop $(docker ps -q)

.github/workflows/codeql-analysis.yml

@@ -17,8 +17,6 @@ on:
   pull_request:
     # The branches below must be a subset of the branches above
     branches: [ master ]
-  schedule:
-    - cron: '44 5 * * 4'
 
 jobs:
   analyze:
@@ -34,34 +32,34 @@ jobs:
         # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
+      - name: Checkout repository
+        uses: actions/checkout@v2
 
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v1
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+          # queries: ./path/to/local/query, your-org/your-repo/queries@main
 
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v1
 
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 https://git.io/JvXDl
 
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
+      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+      #    and modify them (or add more) to build your code if your project
+      #    uses a compiled language
 
-    #- run: |
-    #   make bootstrap
-    #   make release
+      #- run: |
+      #   make bootstrap
+      #   make release
 
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v1

CHANGELOG.md

@@ -1,5 +1,44 @@
 ## Changelog
 
+### 0.16.6
+
+### Fixed
+- Invalid XPath evaluation in HTTP driver [#725](https://github.com/MontFerret/ferret/pull/725)
+- Go routines leakage [#726](https://github.com/MontFerret/ferret/pull/726)
+
+### Updated
+- Small tweak in FQL Parser for FQL Formatter [#723](https://github.com/MontFerret/ferret/pull/723)
+
+### 0.16.5
+
+### Fixed
+- Query fails if an element is not found (regression) [#722](https://github.com/MontFerret/ferret/pull/722)
+
+### Updated
+- Small tweak in FQL Parser for FQL Formatter [#723](https://github.com/MontFerret/ferret/pull/723)
+
+### 0.16.4
+
+### Fixed
+- Fixed inability to parse custom date formats with DATE function [#720](https://github.com/MontFerret/ferret/pull/720)
+
+### 0.16.3
+
+### Fixed
+- Panic during XPath execution by HTTP driver [#715](https://github.com/MontFerret/ferret/pull/715)
+
+### 0.16.2
+
+### Fixed
+- Unable to use dynamic values in LIMIT clause [#706](https://github.com/MontFerret/ferret/pull/706)
+- HTTP driver does not allow to override header values [#707](https://github.com/MontFerret/ferret/pull/707), [#709](https://github.com/MontFerret/ferret/pull/709)
+- Cleaned up deps [#710](https://github.com/MontFerret/ferret/pull/710), [#711](https://github.com/MontFerret/ferret/pull/711)
+
+### 0.16.1
+
+### Fixed
+- Logical precedence in ternary operator condition [#704](https://github.com/MontFerret/ferret/pull/704)
+
 ### 0.16.0
 
 ### Added

Makefile

@@ -1,5 +1,7 @@
 .PHONY: build install compile test e2e doc fmt lint vet release
+export CGO_ENABLED=0
 
+LAB_BIN ?= lab
 DIR_BIN = ./bin
 DIR_PKG = ./pkg
 DIR_E2E = ./e2e
@@ -8,22 +10,27 @@ default: build
 
 build: vet generate test compile
 
+install-tools:
+	go install honnef.co/go/tools/cmd/staticcheck@latest && \
+	go install golang.org/x/tools/cmd/goimports@latest && \
+	go install github.com/mgechev/revive@latest
+
 install:
 	go get
 
 compile:
-	go build -race -v -o ${DIR_BIN}/ferret \
+	go build -v -o ${DIR_BIN}/ferret \
 	${DIR_E2E}/cli.go
 
 test:
-	go test -race ${DIR_PKG}/...
+	go test ${DIR_PKG}/...
 
 cover:
-	go test -race -coverprofile=coverage.txt -covermode=atomic ${DIR_PKG}/... && \
+	go test -coverprofile=coverage.txt -covermode=atomic ${DIR_PKG}/... && \
 	curl -s https://codecov.io/bash | bash
 
 e2e:
-	lab --timeout=120 --attempts=5 --concurrency=1 --wait=http://127.0.0.1:9222/json/version --runtime=bin://./bin/ferret --files=./e2e/tests --cdn=./e2e/pages/dynamic --cdn=./e2e/pages/static
+	${LAB_BIN} --timeout=120 --attempts=5 --concurrency=1 --wait=http://127.0.0.1:9222/json/version --runtime=bin://./bin/ferret --files=./e2e/tests --cdn=./e2e/pages/dynamic --cdn=./e2e/pages/static
 
 bench:
 	go test -run=XXX -bench=. ${DIR_PKG}/...
@@ -36,11 +43,13 @@ doc:
 
 # http://golang.org/cmd/go/#hdr-Run_gofmt_on_package_sources
 fmt:
-	go fmt ${DIR_PKG}/...
+	go fmt ${DIR_PKG}/... && \
+	goimports -w -local github.com/MontFerret ./pkg ./e2e
 
 # https://github.com/mgechev/revive
 # go get github.com/mgechev/revive
 lint:
+	staticcheck ./pkg/compiler ./pkg/drivers ./pkg/runtime ./pkg/stdlib && \
 	revive -config revive.toml -formatter stylish -exclude ./pkg/parser/fql/... -exclude ./vendor/... ./...
 
 # http://godoc.org/code.google.com/p/go.tools/cmd/vet

e2e/cli.go

@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"os/signal"
@@ -114,21 +115,21 @@ func (p *Profiler) Print(label string) {
 	timer, found := p.timers[label]
 
 	if found {
-		fmt.Fprintln(writer, fmt.Sprintf("Time: %s", timer.end.Sub(timer.start)))
+		fmt.Fprintf(writer, "Time: %s\n", timer.end.Sub(timer.start))
 	}
 
 	stats, found := p.allocs[label]
 
 	if found {
-		fmt.Fprintln(writer, fmt.Sprintf("Alloc: %s", byteCountDecimal(stats.Alloc)))
-		fmt.Fprintln(writer, fmt.Sprintf("Frees: %s", byteCountDecimal(stats.Frees)))
-		fmt.Fprintln(writer, fmt.Sprintf("Total Alloc: %s", byteCountDecimal(stats.TotalAlloc)))
-		fmt.Fprintln(writer, fmt.Sprintf("Heap Alloc: %s", byteCountDecimal(stats.HeapAlloc)))
-		fmt.Fprintln(writer, fmt.Sprintf("Heap Sys: %s", byteCountDecimal(stats.HeapSys)))
-		fmt.Fprintln(writer, fmt.Sprintf("Heap Idle: %s", byteCountDecimal(stats.HeapIdle)))
-		fmt.Fprintln(writer, fmt.Sprintf("Heap In Use: %s", byteCountDecimal(stats.HeapInuse)))
-		fmt.Fprintln(writer, fmt.Sprintf("Heap Released: %s", byteCountDecimal(stats.HeapReleased)))
-		fmt.Fprintln(writer, fmt.Sprintf("Heap Objects: %d", stats.HeapObjects))
+		fmt.Fprintf(writer, "Alloc: %s\n", byteCountDecimal(stats.Alloc))
+		fmt.Fprintf(writer, "Frees: %s\n", byteCountDecimal(stats.Frees))
+		fmt.Fprintf(writer, "Total Alloc: %s\n", byteCountDecimal(stats.TotalAlloc))
+		fmt.Fprintf(writer, "Heap Alloc: %s\n", byteCountDecimal(stats.HeapAlloc))
+		fmt.Fprintf(writer, "Heap Sys: %s\n", byteCountDecimal(stats.HeapSys))
+		fmt.Fprintf(writer, "Heap Idle: %s\n", byteCountDecimal(stats.HeapIdle))
+		fmt.Fprintf(writer, "Heap In Use: %s\n", byteCountDecimal(stats.HeapInuse))
+		fmt.Fprintf(writer, "Heap Released: %s\n", byteCountDecimal(stats.HeapReleased))
+		fmt.Fprintf(writer, "Heap Objects: %d\n", stats.HeapObjects)
 	}
 
 	//cpu, found := p.cpus[label]
@@ -138,7 +139,7 @@ func (p *Profiler) Print(label string) {
 	//}
 
 	if writer.Len() > 0 {
-		fmt.Println(fmt.Sprintf("%s:", label))
+		fmt.Printf("%s:\n", label)
 		fmt.Println("-----")
 		fmt.Println(writer.String())
 	}
@@ -260,7 +261,7 @@ func main() {
 		// check whether the app is getting a query via standard input
 		std := bufio.NewReader(os.Stdin)
 
-		b, err := ioutil.ReadAll(std)
+		b, err := io.ReadAll(std)
 
 		if err != nil {
 			fmt.Println(err)
@@ -440,7 +441,7 @@ func execQuery(ctx context.Context, engine *ferret.Instance, opts []runtime.Opti
 		prof.PrintAll()
 
 		if out != nil {
-			fmt.Println(fmt.Sprintf("Output size: %s", byteCountDecimal(uint64(len(out)))))
+			fmt.Printf("Output size: %s\n", byteCountDecimal(uint64(len(out))))
 			fmt.Println("")
 		}
 	}

e2e/tests/dynamic/element/query/element_not_found_by_css.fql

@@ -0,0 +1,8 @@
+LET url = @lab.cdn.dynamic
+LET doc = DOCUMENT(url, { driver: "cdp" })
+
+LET el = ELEMENT(doc, "#do-not-exist")
+
+T::NONE(el)
+
+RETURN TRUE

e2e/tests/dynamic/element/query/element_not_found_by_xpath.fql

@@ -0,0 +1,8 @@
+LET url = @lab.cdn.dynamic
+LET doc = DOCUMENT(url, true)
+
+LET el = ELEMENT(doc, X("//*[@id='do-not-exist']"))?
+
+T::NONE(el)
+
+RETURN TRUE

e2e/tests/dynamic/element/siblings/next_not_found.fql

@@ -0,0 +1,8 @@
+LET doc = DOCUMENT(@lab.cdn.dynamic + "/#/lists", { driver:"cdp" })
+
+LET current = ELEMENT(doc, "body")
+T::NOT::NONE(current)
+LET next = current.nextElementSibling
+T::NONE(next)
+
+RETURN NONE

e2e/tests/dynamic/element/siblings/prev_not_found.fql

@@ -0,0 +1,8 @@
+LET doc = DOCUMENT(@lab.cdn.dynamic + "/#/lists", { driver:"cdp" })
+
+LET current = ELEMENT(doc, 'head')
+T::NOT::NONE(current)
+LET prev = current.previousElementSibling
+T::NONE(prev)
+
+RETURN NONE

e2e/tests/dynamic/element/xpath/attrs.fql

@@ -0,0 +1,8 @@
+LET url = @lab.cdn.dynamic
+LET page = DOCUMENT(url, true)
+
+LET actual = XPATH(page, "//body/@class")
+
+T::NOT::EMPTY(actual)
+
+RETURN T::EQ(actual[0], "text-center")

examples/embedded/main.go

@@ -27,7 +27,7 @@ func main() {
 	}
 
 	for _, topic := range topics {
-		fmt.Println(fmt.Sprintf("%s: %s %s", topic.Name, topic.Description, topic.URL))
+		fmt.Printf("%s: %s %s\n", topic.Name, topic.Description, topic.URL)
 	}
 }
 

examples/xpath.fql

@@ -0,0 +1,5 @@
+LET doc = DOCUMENT("https://www.google.ca", {
+  driver: 'cdp'
+})
+
+RETURN XPATH(doc, "//meta/@charset") // ["UTF-8"]