fix: prompt in LLMEvaluator (#31)

dynamiq-ai · Nov 7, 2024 · b8093aa · b8093aa · github-actions · Nov 7, 2024
1 parent 7d2dfe8
commit b8093aa
Show file tree

Hide file tree

Showing 5 changed files with 21 additions and 19 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -28,7 +28,7 @@ jobs:
           envkey_PINECONE_REGION: ${{ secrets.PINECONE_REGION }}
 
       - name: Build
-        uses: docker/bake-action@v4
+        uses: docker/bake-action@v5
         with:
           set: |
             *.cache-from=type=gha
@@ -77,7 +77,7 @@ jobs:
           envkey_PINECONE_REGION: ${{ secrets.PINECONE_REGION }}
 
       - name: Load Image
-        uses: docker/bake-action@v4
+        uses: docker/bake-action@v5
         with:
           load: true
           set: |

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.12.2-slim as runtime
+FROM python:3.12.2-slim AS runtime
 
 ENV PYTHONPATH=/app/
 ENV RUNTIME_PACKAGES="git curl make"

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -1,27 +1,28 @@
-version: "3.8"
-
-x-base-app-test: &base-app-test
-  image: dynamiq-app:${IMAGE_TAG:-local}
-  build:
-    target: develop
-  volumes:
-    - ./:/app
-
 services:
   dynamiq-app:
-    # Caching target to avoid cache duplication
-    <<: *base-app-test
+    image: dynamiq-app:${IMAGE_TAG:-local}
+    build:
+      target: develop
+    volumes:
+      - ./:/app
 
-  dynamiq-app-test: &app-test
-    <<: *base-app-test
+  dynamiq-app-test:
+    image: dynamiq-app:${IMAGE_TAG:-local}
+    build:
+      target: develop
     entrypoint: ["make", "test"]
     env_file:
       - .env
+    volumes:
+      - ./:/app
 
   dynamiq-app-test-cov:
-    <<: *app-test
+    image: dynamiq-app:${IMAGE_TAG:-local}
+    build:
+      target: develop
     entrypoint: ["make", "test-cov"]
     env_file:
       - .env
     volumes:
+      - ./:/app
       - ./reports:/app/reports:wr
diff --git a/dynamiq/components/evaluators/llm_evaluator.py b/dynamiq/components/evaluators/llm_evaluator.py
@@ -266,7 +266,8 @@ def prepare_template(self) -> str:
         return (
             f"Instructions:\n"
             f"{self.instructions}\n\n"
-            f"Generate the response in JSON format with the following keys:\n"
+            f"Generate the response in JSON format omit extra keys and markdown syntax elements, "
+            f"and include the following keys:\n"
             f"{json.dumps(self.outputs)}\n"
             f"Consider the instructions and the examples below to determine those values.\n\n"
             f"Examples:\n"

diff --git a/pyproject.toml b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "dynamiq"
-version = "0.3.2"
+version = "0.3.3"
 description = "Dynamiq is an orchestration framework for agentic AI and LLM applications"
 authors = ["Dynamiq Team <[email protected]>"]
 license = "Apache License 2.0"
File	Stmts	Miss	Cover	Missing
dynamiq
__init__.py	3	0	100%
dynamiq/cache
__init__.py	1	0	100%
codecs.py	11	2	81%	16, 27
config.py	14	1	92%	34
utils.py	26	0	100%
dynamiq/cache/backends
__init__.py	2	0	100%
base.py	23	5	78%	34, 47, 59, 72, 84
redis.py	16	4	75%	20, 22, 48, 59
dynamiq/cache/managers
__init__.py	2	0	100%
base.py	39	4	89%	117–118, 120, 146
workflow.py	28	2	92%	73–74
dynamiq/callbacks
__init__.py	3	0	100%
base.py	49	12	75%	19, 31, 43, 79, 115, 132, 168, 187, 205, 209–210, 212
streaming.py	55	15	72%	96, 126–127, 135, 160–161, 169–174, 182, 190–191
tracing.py	169	4	97%	64, 120, 485, 507
dynamiq/clients
__init__.py	1	0	100%
base.py	8	2	75%	5, 21
dynamiq/components
__init__.py	0	0	100%
serializers.py	27	9	66%	25, 40, 58, 70, 101, 120, 122, 134, 136
dynamiq/components/converters
__init__.py	0	0	100%
base.py	49	15	69%	43, 48–50, 53–54, 61–62, 64–65, 98, 102, 107, 118, 122
pptx.py	34	9	73%	59–61, 66, 96–97, 101–102, 104
pypdf.py	60	20	66%	71–72, 77, 100–102, 125–135, 137–138, 140
unstructured.py	116	48	58%	63, 69–70, 82–83, 90, 109, 143, 194–195, 200, 220–221, 228–229, 232, 259–260, 263, 283, 292–302, 304–305, 307, 312–325
utils.py	11	4	63%	23–25, 27
dynamiq/components/embedders
__init__.py	0	0	100%
base.py	71	7	90%	76, 80, 137–138, 159, 163, 167
bedrock.py	18	4	77%	19, 26–28
cohere.py	18	2	88%	21, 29
huggingface.py	10	1	90%	19
mistral.py	10	1	90%	19
openai.py	16	2	87%	44, 51
watsonx.py	10	1	90%	19
dynamiq/components/retrievers
__init__.py	0	0	100%
chroma.py	19	2	89%	33–34
pinecone.py	17	2	88%	34–35
qdrant.py	17	2	88%	31–32
weaviate.py	19	2	89%	31–32
dynamiq/components/splitters
__init__.py	0	0	100%
document.py	55	7	87%	60, 63, 83, 88, 97, 118, 124
dynamiq/connections
__init__.py	2	0	100%
connections.py	335	44	86%	13–17, 99, 107, 118, 141, 528–529, 531–532, 536–537, 539–540, 553–554, 556, 584, 586, 598, 600–602, 643–644, 666–667, 673–675, 677–680, 682, 699–700, 737–738, 746, 854
managers.py	60	5	91%	70–71, 107, 167, 179
storages.py	14	1	92%	51
dynamiq/executors
__init__.py	0	0	100%
base.py	12	2	83%	35, 58
pool.py	60	11	81%	90, 111–112, 115, 163–164, 179, 192, 203–204, 215
dynamiq/flows
__init__.py	2	0	100%
base.py	25	1	96%	35
flow.py	127	31	75%	68, 75, 82, 116, 159–161, 246–247, 249, 268–269, 271, 274, 293–294, 297–298, 301–302, 304–305, 308–310, 312–316, 318
dynamiq/loaders
__init__.py	0	0	100%
yaml.py	300	90	70%	55, 58, 62–63, 66–69, 72, 97, 101, 112–113, 137–138, 157, 186, 213, 237, 240, 243, 246, 266–270, 273, 293–297, 300–301, 327–328, 333–334, 339–340, 382, 385–386, 397–400, 413–414, 452, 455, 458, 492, 496, 514–515, 599–600, 602–603, 607–610, 612, 617, 656, 659, 666, 674, 687–688, 720, 722, 725, 730, 757, 761, 765, 769–770, 823–824, 853, 902–906
dynamiq/memory
__init__.py	1	0	100%
memory.py	59	28	52%	17, 33–35, 50, 53, 56, 60–61, 67–77, 79, 83, 87–92
dynamiq/memory/backend
__init__.py	5	0	100%
base.py	19	5	73%	14, 19, 24, 29, 34
in_memory.py	70	46	34%	19–22, 26–27, 31, 35–39, 41–49, 51, 67–68, 76–81, 83–84, 88–89, 91–100, 104, 108
pinecone.py	76	55	27%	34–37, 39–40, 54–55, 57–58, 62, 71–73, 77–80, 82, 84–85, 89–94, 98–99, 101–106, 110–111, 113–115, 122–124, 132, 134–136, 140–143, 147–150
qdrant.py	85	63	25%	41–43, 45–46, 55–59, 63, 72–74, 85–88, 90, 93–94, 105–110, 123–127, 133–136, 138, 140–142, 146–147, 149–156, 158, 160–161, 165–170, 174–177
sqlite.py	119	94	21%	50–51, 53–56, 60–61, 63–67, 69–71, 73, 75–76, 80–87, 91–96, 106, 108–109, 113–119, 124–125, 129–135, 137–138, 142–149, 153–156, 158–160, 162–164, 166–168, 171–173, 175–176, 178–182, 184–187, 189, 191–192, 194–195
dynamiq/nodes
__init__.py	2	0	100%
exceptions.py	15	1	93%	4
managers.py	12	7	41%	28–34
node.py	331	44	86%	248, 259, 276, 281–282, 286–287, 310–311, 313, 328, 358, 360, 367–368, 568–570, 639–645, 647–652, 656, 658, 660, 830, 848, 856, 859, 877–879, 957, 982, 988
types.py	23	0	100%
dynamiq/nodes/agents
__init__.py	4	0	100%
base.py	251	99	60%	82, 86–91, 97, 100–102, 128, 132, 157–158, 174, 189–190, 196–200, 220, 222–223, 226, 233–235, 239–240, 245–249, 251–252, 254–257, 269–270, 274–276, 282, 286–289, 291, 297–301, 303–304, 321–326, 332, 337, 370–371, 375–376, 380, 384, 388, 394–397, 401–403, 407, 409–410, 412–413, 415, 420–421, 428, 431, 435–436, 440–441, 445–446
exceptions.py	15	3	80%	7–8, 61
react.py	164	126	23%	177, 226, 267–268, 272–274, 276–279, 287, 291–292, 294–295, 297, 300–302, 309, 329–330, 332–333, 339, 341, 343–349, 352, 364, 366–367, 371–372, 374–379, 383–387, 389, 391–393, 396–397, 399–402, 404, 407, 411–413, 416–417, 419–422, 424–431, 433–435, 437–440, 442, 446–447, 449, 451, 460–461, 466, 468–474, 478, 480, 488–491, 493, 497–499, 503, 505, 513–523, 525
reflection.py	36	24	33%	43–44, 47–48, 51, 64–66, 68–71, 73, 76–77, 80–81, 83–85, 87–90
simple.py	3	0	100%
dynamiq/nodes/audio
__init__.py	2	0	100%
elevenlabs.py	76	6	92%	39, 41, 89, 127, 172, 214
whisper.py	47	8	82%	46, 72, 79–80, 82, 108, 124, 126
dynamiq/nodes/converters
__init__.py	4	0	100%
llm_text_extractor.py	173	121	30%	14, 49–53, 107–108, 114, 124, 132–134, 145–147, 175–177, 179, 187, 210–211, 213, 215–218, 225–226, 233–234, 236–239, 241–243, 245–246, 248–253, 255, 267, 269, 290, 292, 294–296, 298, 302, 326–327, 332, 334–335, 344, 348, 363, 370, 372–375, 389–390, 392–397, 410, 433–439, 442–443, 496, 498–500, 527–529, 531, 539, 562–563, 565, 567–570, 577–578, 585–586, 588–591, 593–594, 596–601, 603
pptx.py	31	1	96%	33
pypdf.py	32	1	96%	34
unstructured.py	39	1	97%	58
dynamiq/nodes/embedders
__init__.py	6	0	100%
bedrock.py	53	2	96%	48, 135
cohere.py	53	2	96%	48, 136
huggingface.py	53	2	96%	48, 137
mistral.py	53	2	96%	49, 137
openai.py	55	2	96%	53, 151
watsonx.py	53	2	96%	46, 130
dynamiq/nodes/llms
__init__.py	19	0	100%
ai21.py	9	1	88%	24
anthropic.py	8	1	87%	22
anyscale.py	9	1	88%	24
azureai.py	9	1	88%	24
base.py	104	8	92%	13, 173, 231–233, 235–237
bedrock.py	9	1	88%	24
cerebras.py	9	1	88%	24
cohere.py	8	3	62%	21–23
custom_llm.py	4	0	100%
deepinfra.py	9	1	88%	24
gemini.py	21	1	95%	43
groq.py	9	1	88%	24
huggingface.py	9	1	88%	24
mistral.py	9	1	88%	24
openai.py	8	1	87%	22
replicate.py	9	1	88%	24
sambanova.py	9	3	66%	23–25
togetherai.py	9	1	88%	24
watsonx.py	9	1	88%	24
dynamiq/nodes/operators
__init__.py	1	0	100%
operators.py	144	40	72%	158, 174, 176, 178, 180, 184–191, 193, 219–220, 222–224, 226–229, 231, 233–236, 238, 242, 266, 284–288, 290, 292–293, 295
dynamiq/nodes/retrievers
__init__.py	4	0	100%
chroma.py	37	0	100%
pinecone.py	41	0	100%
qdrant.py	37	0	100%
weaviate.py	37	0	100%
dynamiq/nodes/splitters
__init__.py	1	0	100%
document.py	30	0	100%
dynamiq/nodes/tools
__init__.py	8	0	100%
e2b_sandbox.py	155	118	23%	63, 77–78, 80–81, 113–115, 117, 127, 139–142, 146–151, 155–157, 161–163, 167–170, 172–173, 175, 177–178, 185, 187–188, 192–193, 196–201, 203, 208–209, 211, 215–219, 224, 228–235, 239–241, 245–246, 248–252, 256, 258–259, 261–262, 264–282, 288–290, 292–302, 304–305, 309–312
firecrawl.py	68	36	47%	59–60, 64–65, 73–74, 76, 82, 86–87, 89–92, 94, 105–106, 110–111, 117–120, 123, 125, 127–134, 138, 150, 152
http_api_call.py	47	5	89%	69, 81, 90, 99–100
llm_summarizer.py	55	29	47%	92–94, 100, 110, 122–124, 141–142, 148–151, 171–173, 175–176, 180–181, 186–188, 192–193, 195, 197, 201
python.py	51	16	68%	40–42, 81, 85–86, 149, 153–157, 173, 187–189
scale_serp.py	61	44	27%	56–57, 59–64, 66–67, 76, 91, 95–96, 98–100, 102–103, 107, 109, 111–112, 117–120, 123, 125, 127, 129–135, 137, 142–143, 151, 153, 160, 162
tavily.py	58	33	43%	78–86, 88, 104, 108–109, 111–112, 125, 127–128, 133–136, 139, 141, 143, 145, 149–150, 157–158, 161, 171, 173
zenrows.py	35	19	45%	50, 55–56, 58–60, 62, 67, 69, 74–77, 80, 82–83, 85–86, 88
dynamiq/nodes/utils
__init__.py	1	0	100%
utils.py	11	0	100%
dynamiq/nodes/validators
__init__.py	5	0	100%
base.py	22	5	77%	37–40, 45
regex_match.py	14	1	92%	37
valid_choices.py	9	1	88%	31
valid_json.py	10	2	80%	27–28
valid_python.py	8	2	75%	23–24
dynamiq/nodes/writers
__init__.py	4	0	100%
chroma.py	27	10	62%	37–39, 43, 47, 69–70, 72, 74–75
pinecone.py	40	4	90%	41, 54, 57, 60
qdrant.py	28	1	96%	38
weaviate.py	29	11	62%	38–40, 44, 48, 69–70, 72, 74–75, 77
dynamiq/prompts
__init__.py	1	0	100%
prompts.py	84	15	82%	98, 141, 154, 197–201, 206–207, 216–217, 221, 223, 230
dynamiq/runnables
__init__.py	1	0	100%
base.py	45	1	97%	145
dynamiq/storages
__init__.py	0	0	100%
dynamiq/storages/vector
__init__.py	4	0	100%
base.py	5	0	100%
exceptions.py	6	0	100%
policies.py	6	0	100%
utils.py	2	0	100%
dynamiq/storages/vector/chroma
__init__.py	1	0	100%
chroma.py	186	149	19%	10–11, 56–57, 60, 71, 89–91, 94, 96, 98–99, 101–102, 104, 106, 117–119, 123–124, 128, 140–141, 143–144, 154–155, 176–177, 183–184, 192, 237–239, 241–244, 246, 248, 252, 261–262, 284–285, 288–289, 292, 294–295, 298–300, 302–304, 306, 308–315, 317–319, 321–323, 325–327, 329, 338–341, 343–358, 360–361, 363–364, 366, 382–384, 387, 389, 402–403, 407, 423–425, 427, 429–430, 432, 446–449, 451–454, 459–460, 462–463, 465–466, 468–469, 471, 485–488, 490–491, 496–497, 499–500, 502–503, 505–506
dynamiq/storages/vector/pinecone
__init__.py	1	0	100%
filters.py	95	80	15%	22–24, 26–28, 45–50, 52–53, 55–56, 58–59, 75, 78, 80–88, 93, 95, 97, 114–116, 120, 122, 139–141, 145, 147, 164–166, 170, 172, 189–191, 195, 197, 214–216, 220, 222, 239–241, 245, 247, 264–265, 268, 270–273, 277, 279, 296–298, 300–303, 307, 309
pinecone.py	164	59	64%	17, 81–83, 117, 131, 143, 145–146, 148, 161–163, 170, 172, 190–192, 197, 201–202, 212–214, 216–217, 221, 233, 237, 243–244, 246–247, 257–258, 271–273, 275–277, 279–281, 283, 290, 292–293, 302–303, 306–308, 324–325, 354, 357, 396–397
dynamiq/storages/vector/qdrant
__init__.py	1	0	100%
converters.py	41	13	68%	26, 28–30, 32–35, 67–68, 70–71, 75
filters.py	133	33	75%	61–62, 65–66, 80–81, 91–92, 99, 111, 177–178, 186–187, 217–218, 227, 233–234, 261–262, 277, 282–283, 288, 293–294, 299, 304–305, 310, 315–316
qdrant.py	267	75	71%	29, 240, 256, 280, 284, 302–303, 306, 335–336, 339–340, 380–381, 445, 485, 518–519, 523, 525–528, 540, 544–549, 591–594, 596–597, 633–634, 638, 640, 642–643, 666–668, 670, 672, 688–689, 694, 747–748, 761, 763, 768–769, 777, 779–780, 785, 787–789, 791–792, 794–795, 801, 803–804, 810, 843, 845, 897–898
dynamiq/storages/vector/weaviate
__init__.py	1	0	100%
filters.py	126	105	16%	22–24, 26–28, 56–63, 65, 87–92, 94–99, 101–105, 107–108, 121–126, 140–142, 156–157, 159, 178–184, 188–189, 206–212, 216–217, 234–240, 244–245, 262–268, 272–273, 290–292, 294, 311–315, 343, 345, 351, 353–360, 362, 377–378
weaviate.py	165	127	23%	17–18, 57–59, 67–68, 70, 80–81, 90–91, 103–105, 107–108, 110–111, 113, 125–126, 128, 130–133, 135, 137–139, 141–145, 147–148, 150, 158, 160, 172–174, 177–180, 195, 197–200, 203–204, 211–216, 228–229, 231–232, 244–245, 249–251, 267–271, 273, 279–284, 286, 292, 294, 311–316, 318, 321, 323–324, 330–337, 352–353, 355, 368–371, 373–375, 386–387, 389, 398–399, 418–419, 429, 457–459, 461–462, 473
dynamiq/types
__init__.py	1	0	100%
document.py	16	0	100%
streaming.py	31	3	90%	48, 56, 84
dynamiq/utils
__init__.py	2	0	100%
duration.py	11	1	90%	45
env.py	8	0	100%
jsonpath.py	46	14	69%	18–19, 41, 43, 50–51, 55, 59–61, 84, 94–96
logger.py	10	0	100%
utils.py	60	7	88%	32, 34, 94–96, 148–149
dynamiq/workflow
__init__.py	1	0	100%
workflow.py	67	17	74%	15, 31, 34, 55, 57–58, 61–63, 65, 82–83, 86, 91–94
examples
__init__.py	0	0	100%
examples/rag
__init__.py	0	0	100%
dag_yaml.py	37	8	78%	65–67, 74–75, 81, 86, 88
utils.py	19	0	100%
tests
__init__.py	0	0	100%
conftest.py	78	1	98%	83
tests/integration
__init__.py	0	0	100%
tests/integration/flows
__init__.py	0	0	100%
tests/integration/nodes
__init__.py	0	0	100%
tests/integration/nodes/audio
__init__.py	0	0	100%
tests/integration/nodes/llms
__init__.py	0	0	100%
tests/integration/nodes/operators
__init__.py	0	0	100%
tests/integration/nodes/tools
__init__.py	0	0	100%
tests/integration/nodes/validators
__init__.py	0	0	100%
tests/integration_with_creds
__init__.py	0	0	100%
TOTAL	7073	2233	68%