From 4a127e0c65f92e8763f658a52448bea4fd5c58a1 Mon Sep 17 00:00:00 2001 From: raghavdixit99 Date: Thu, 18 Apr 2024 11:26:08 -0400 Subject: [PATCH 1/8] integration update --- .../integrations/vectorstores/lancedb.ipynb | 98 +++++++--- .../modules/data_connection/indexing.ipynb | 2 +- .../vectorstores/lancedb.py | 173 +++++++++++++----- .../vectorstores/test_lancedb.py | 18 +- .../vectorstores/test_indexing_docs.py | 1 + 5 files changed, 215 insertions(+), 77 deletions(-) diff --git a/docs/docs/integrations/vectorstores/lancedb.ipynb b/docs/docs/integrations/vectorstores/lancedb.ipynb index 9d9fcbdbdecc4..9928fcdbc4347 100644 --- a/docs/docs/integrations/vectorstores/lancedb.ipynb +++ b/docs/docs/integrations/vectorstores/lancedb.ipynb @@ -12,6 +12,16 @@ "This notebook shows how to use functionality related to the `LanceDB` vector database based on the Lance data format." ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "88ac92c0", + "metadata": {}, + "outputs": [], + "source": [ + "! pip install -U langchain-openai" + ] + }, { "cell_type": "code", "execution_count": null, @@ -32,7 +42,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 2, "id": "a0361f5c-e6f4-45f4-b829-11680cf03cec", "metadata": { "tags": [] @@ -47,26 +57,15 @@ }, { "cell_type": "code", - "execution_count": 10, - "id": "aac9563e", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "from langchain.embeddings import OpenAIEmbeddings\n", - "from langchain.vectorstores import LanceDB" - ] - }, - { - "cell_type": "code", - "execution_count": 11, + "execution_count": 13, "id": "a3c3999a", "metadata": {}, "outputs": [], "source": [ "from langchain.document_loaders import TextLoader\n", "from langchain_text_splitters import CharacterTextSplitter\n", + "from langchain_openai import OpenAIEmbeddings\n", + "from langchain.vectorstores import LanceDB\n", "\n", "loader = TextLoader(\"../../modules/state_of_the_union.txt\")\n", "documents = loader.load()\n", @@ -75,22 +74,69 @@ "embeddings = OpenAIEmbeddings()" ] }, + { + "cell_type": "markdown", + "id": "e9517bb0", + "metadata": {}, + "source": [ + "##### For LanceDB cloud, you can invoke the vector store as follows :\n", + "\n", + "\n", + "```python\n", + "db_url = \"db://lang_test\" # url of db you created\n", + "api_key = \"xxxxx\" # your API key\n", + "region=\"us-east-1-dev\" # your selected region\n", + "\n", + "vector_store = LanceDB(\n", + " uri=db_url,\n", + " api_key=api_key,\n", + " region=region,\n", + " embedding=embeddings,\n", + " table_name='langchain_test'\n", + " )\n", + "```\n" + ] + }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 7, "id": "6e104aee", "metadata": {}, "outputs": [], "source": [ "docsearch = LanceDB.from_documents(documents, embeddings)\n", - "\n", "query = \"What did the president say about Ketanji Brown Jackson\"\n", "docs = docsearch.similarity_search(query)" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 8, + "id": "f5e1cdfd", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tbl: LanceTable(connection=LanceDBConnection(/tmp/lancedb), name=\"vectorstore\")\n" + ] + } + ], + "source": [ + "# to explore the table, you can load it into a df or save it in a csv file: \n", + "tbl = docsearch.get_table()\n", + "print(\"tbl:\", tbl)\n", + "pd_df = tbl.to_pandas()\n", + "# pd_df.to_csv(\"docsearch.csv\", index=False)\n", + "\n", + "# you can also create a new vector store object using an older connection object:\n", + "vector_store = LanceDB(connection=tbl, embedding=embeddings)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, "id": "9c608226", "metadata": {}, "outputs": [ @@ -166,7 +212,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 10, "id": "a359ed74", "metadata": {}, "outputs": [ @@ -242,10 +288,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "9d749a3f-df17-4a8a-b256-08a3bbc74cb6", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'vector': [-0.005857204552739859, -0.0019384637707844377, -0.0046228766441345215, -0.002721208380535245, -0.0009709044243209064, 0.019776010885834694, 0.017059819772839546, 0.008590120822191238, -0.019227420911192894, -0.016792215406894684, 0.01087145321071148, 0.010985185392200947, -0.0004419664037413895, -0.007513010874390602, -0.00359259732067585, -0.0006225997931323946, 0.03334358334541321, -0.009974977001547813, 0.00763343321159482, -0.030667534098029137, -0.013835178688168526, 0.0036461183335632086, 0.014664754271507263, 0.006733611226081848, 0.0008337568724527955, 0.010536947287619114, 0.03179147467017174, -0.009807723574340343, 0.011038706637918949, -0.018531646579504013, -0.017300663515925407, 0.006057908292859793, -0.03676892817020416, -0.018678829073905945, -0.027804160490632057, -0.006138190161436796, 9.779081301530823e-05, -0.011145749129354954, 0.01878587156534195, -0.013848558999598026, 0.03588583320379257, -0.02578374370932579, 0.002217776607722044, -0.018732350319623947, -0.01890629343688488, 0.018197141587734222, -0.009433076716959476, -0.020511923357844353, -0.02618515118956566, 0.025623179972171783, 0.04444919154047966, 0.0018464745953679085, -0.028901340439915657, -0.024914026260375977, 0.011841521598398685, -0.013754897750914097, -0.013052434660494328, 0.00946652702987194, -0.010978495702147484, -0.01161405723541975, 0.013848558999598026, 0.0013087557163089514, -0.02234501764178276, 0.01692601665854454, -0.00963378045707941, -0.018491506576538086, 0.001245199586264789, 0.021903470158576965, -0.003853512229397893, 0.015721794217824936, 0.05437733978033066, 0.0028633736073970795, 0.02597106620669365, -0.006847343407571316, 0.031523872166872025, -0.01960206776857376, -0.004994178656488657, -0.004813545383512974, -0.0010010099504143, -0.017862634733319283, 0.022478820756077766, -0.012958772480487823, -0.022425299510359764, 0.01635066606104374, 0.013139406219124794, -0.014785177074372768, -0.0023281637113541365, -0.001144847716204822, 0.0027145184576511383, 0.0005193209508433938, 0.013012293726205826, 0.02453937940299511, -0.00315104890614748, 0.0066098435781896114, -0.00980103388428688, 0.0149457398802042, -0.026961205527186394, 0.021716147661209106, 0.007907727733254433, -0.026265431195497513, 0.016752073541283607, -0.020806290209293365, -0.02623867057263851, -0.011165819130837917, -0.028847821056842804, -0.029784437268972397, -0.00617164047434926, 0.002262934809550643, 0.004475693684071302, 0.011045397259294987, -0.006797167472541332, 0.009292583912611008, 0.0015537815634161234, 0.0074996305629611015, -0.011687648482620716, -0.02189009077847004, -0.018009817227721214, -0.025797123089432716, 0.022291498258709908, -0.03912385180592537, 0.018799252808094025, 0.014611233957111835, 0.0021006993483752012, -0.028045006096363068, 0.01972248964011669, -0.01990981213748455, -0.013888699933886528, 0.008242234587669373, 0.0006455971160903573, 0.004000694956630468, -0.011547156609594822, -0.011567226611077785, 0.02365628257393837, -0.008041530847549438, 0.011560536921024323, 0.007981319911777973, -0.020873190835118294, 0.01750136725604534, -0.013687996193766594, -0.01833094283938408, 0.019200660288333893, -0.01310595590621233, -0.025596419349312782, -0.015521090477705002, 0.012577435933053493, 0.021381640806794167, 0.0004883791552856565, 0.005743472371250391, 0.014731655828654766, 0.0159893985837698, 0.015012641437351704, -0.016390807926654816, -0.00020749842224176973, -0.007934488356113434, 0.005034319125115871, -0.002130804816260934, -0.024365436285734177, 0.0018381119007244706, 0.008837655186653137, 0.0027914547827094793, 0.006389069836586714, -0.019802771508693695, -0.009861244820058346, 0.03650132194161415, 0.023268256336450577, 0.016430947929620743, -0.014557712711393833, 0.027054866775870323, -0.0006815565284341574, 0.025502758100628853, 0.019200660288333893, 0.017648549750447273, -0.015240105800330639, 0.02783092111349106, 0.0004156240320298821, 0.007807376328855753, 0.018277421593666077, -0.008623571135103703, -0.027135148644447327, 0.01732742413878441, 0.01814362034201622, 0.0009550153627060354, 0.012149267829954624, 0.04378018155694008, -0.02301403135061264, -0.009433076716959476, 0.01756826974451542, 0.007599982433021069, -0.023388678207993507, -0.007506320718675852, -0.0012585797812789679, 0.020859811455011368, 0.011306311935186386, -0.012383421882987022, -0.6452491879463196, -0.01450419146567583, 0.017394326627254486, -0.011814760975539684, -0.0038668925408273935, 0.010175680741667747, -0.0010461683850735426, 0.01756826974451542, -0.016430947929620743, 0.015494329854846, -0.013400320895016193, 0.012724618427455425, 0.0006355618825182319, -0.02286684885621071, -0.02635909430682659, -0.02809852547943592, -0.013206307776272297, -0.008282375521957874, 0.031095702201128006, -0.006268647499382496, -0.026385854929685593, 0.004224814008921385, -0.005940831266343594, 0.007392588537186384, 0.008248924277722836, 0.015547851100564003, 0.020525304600596428, -0.016096441075205803, -0.015467570163309574, 0.010570397600531578, -0.029971761628985405, 0.03229992464184761, -0.0016432619886472821, 0.005756852682679892, 0.04000695049762726, -0.0024970893282443285, 0.0032865239772945642, 0.022733045741915703, 0.013467222452163696, 0.021729527041316032, -0.012684477493166924, 0.024740083143115044, 0.03963230177760124, -0.0027312436141073704, 0.025489376857876778, -0.005589599721133709, 0.021970370784401894, -0.009981666691601276, 0.0006426701438613236, 0.02401755005121231, -0.005766888149082661, 0.019521785899996758, -0.0006159096374176443, 0.0012109127128496766, -0.016056301072239876, -0.002257917309179902, -0.006870758719742298, -0.021328119561076164, 0.000647269596811384, 0.0023248186334967613, -0.0060612536035478115, 0.008456318639218807, -0.0227062851190567, -0.011045397259294987, -0.019628828391432762, 0.02855345420539379, 0.012477084062993526, 0.011814760975539684, 0.019013335928320885, -0.006593118421733379, 0.014544332399964333, 0.004375341814011335, -0.010316173546016216, -0.035270340740680695, 0.010102089494466782, -0.004121117293834686, 0.026278812438249588, -0.013279899023473263, -0.005756852682679892, 0.01133976224809885, 0.0014149615308269858, -0.005489247851073742, -0.00027095008408650756, -0.0010821277974173427, 0.016270384192466736, 0.008984838612377644, -0.021542202681303024, 0.005803683772683144, 0.003796646138653159, 0.012122507207095623, 0.025382336229085922, 0.0023482339456677437, 0.010965115390717983, -0.01787601411342621, 0.01087145321071148, 0.022505581378936768, -0.023549241945147514, 0.002558972919359803, 0.012176028452813625, -0.03163091093301773, -0.0032681261654943228, -0.011988705024123192, 0.0238034650683403, 0.007158434018492699, 0.016645031049847603, 0.013179547153413296, -0.013848558999598026, -0.010276032611727715, 0.03626047819852829, -0.03842807933688164, -0.015293626114726067, 0.0055929445661604404, -0.018732350319623947, -0.017487987875938416, 0.006559667643159628, -0.016551369801163673, 0.007686953991651535, 0.018892914056777954, -0.0149457398802042, -0.019976714625954628, 0.005071115214377642, -0.00025464288773946464, 0.012958772480487823, -0.03899005055427551, 0.0154809495434165, 0.004780094604939222, 0.006506146863102913, -0.0030222642235457897, -0.0009215647587552667, -0.01061053853482008, 0.0033132846001535654, -0.004298405721783638, 0.02079290896654129, 0.008998218923807144, 0.015186584554612637, -0.004823580384254456, 0.01404926273971796, -0.011025326326489449, 0.016966158524155617, -0.011426733806729317, -0.0029319473542273045, -0.0014124526642262936, -0.0060980492271482944, -0.0042482297867536545, -0.024726703763008118, -0.026278812438249588, -0.0006384888547472656, -0.0026676873676478863, -0.030185844749212265, -0.027563316747546196, 0.010068638250231743, -0.018678829073905945, -0.02167600579559803, -0.006389069836586714, 0.007412659004330635, -0.0011314674047753215, -0.0154809495434165, 0.015601372346282005, -0.004689777735620737, -0.021970370784401894, 0.015788694843649864, 0.003529041074216366, -0.027964724227786064, -0.014437290839850903, 0.009834484197199345, -0.010389764793217182, 0.041023846715688705, 0.013192927464842796, 0.006753681693226099, -0.0276168379932642, 0.0009893022943288088, -0.013888699933886528, -0.0014283417258411646, -0.0009951561223715544, 0.037678785622119904, 0.016698552295565605, -0.028714017942547798, 0.013587644323706627, 0.016551369801163673, 0.011701028794050217, -0.007278856355696917, -0.012891871854662895, -0.025195011869072914, -0.006971110589802265, 0.01847812533378601, 0.016270384192466736, 0.004074286203831434, 0.015052782371640205, -0.024378817528486252, 0.0209802333265543, -0.0007543116225861013, -0.00763343321159482, -0.01823728159070015, 0.009861244820058346, 0.012697857804596424, 0.03026612661778927, 0.008864415809512138, 0.00574681768193841, 0.0033015769440680742, 0.029061904177069664, 0.010878143832087517, 0.011045397259294987, 0.022077413275837898, -0.010222511366009712, -0.018290802836418152, -0.038374558091163635, 0.009794343262910843, -0.031737953424453735, 0.02131473831832409, 0.002231156686320901, -0.005659846123307943, -0.027777399867773056, -0.026318952441215515, -0.03238020837306976, 0.009640470147132874, 0.01859854906797409, -0.009727442637085915, 0.030373169109225273, -0.010202441364526749, -0.016645031049847603, 0.011406663805246353, 0.017059819772839546, 0.01723376289010048, -0.00834258645772934, 0.004947347566485405, 0.0027864372823387384, -0.001956861699000001, 0.01206898596137762, 0.01095842570066452, -0.0014609560603275895, 0.00033220654586330056, 0.018585167825222015, 0.015641512349247932, 0.031523872166872025, -0.00989469513297081, -0.021836569532752037, 0.002165928017348051, -0.0015253485180437565, 0.04182666167616844, 0.004666362423449755, 0.02609148807823658, 0.02015065774321556, 0.016631651669740677, -0.0008906229049898684, 0.025649940595030785, -0.031764715909957886, 0.0443153902888298, 0.009727442637085915, 0.030373169109225273, 0.013875319622457027, 0.0003307430597487837, 0.002821560250595212, -0.0011047068983316422, 0.008416177704930305, 0.01726052351295948, -0.03521681949496269, -0.0022930405102670193, 0.020833050832152367, 0.014758416451513767, 0.02587740495800972, 0.03355766832828522, 0.030399929732084274, 0.016537990421056747, -0.017407706007361412, -0.0074594896286726, 0.0013162821996957064, -0.010302793234586716, 0.007907727733254433, -0.004703158047050238, -0.018103478476405144, 0.0020789564587175846, -0.017956295982003212, 0.010282722301781178, 0.00774047477170825, 0.031523872166872025, 0.02310769259929657, -0.004736608825623989, 0.0149457398802042, 0.0023800120688974857, 0.003452104749158025, -0.00919223204255104, -0.033744990825653076, 0.027228809893131256, 0.02399078942835331, -0.014905598945915699, -0.003629392944276333, -0.010557017289102077, -0.021635865792632103, -0.02119431644678116, 0.019682347774505615, -0.01738094538450241, -0.007640122901648283, -0.008931317366659641, 0.0298647191375494, -0.004920586943626404, -0.016216862946748734, 0.030587252229452133, -0.013861939311027527, 0.01759503036737442, -0.0014300142647698522, 0.005178156774491072, -0.0013689668849110603, -0.022933749482035637, -0.0039438288658857346, 0.008824274875223637, -0.0030925103928893805, 0.012597505934536457, -0.012015464715659618, 0.023549241945147514, -0.018183760344982147, 0.018183760344982147, -0.009760892949998379, -0.013360179960727692, 0.010664059780538082, 0.002741278847679496, -0.0038066813722252846, -0.027349231764674187, -0.014209826476871967, 0.004395412281155586, -0.011982014402747154, -0.0020806288812309504, -0.024191493168473244, -0.04469003528356552, -0.016658412292599678, 0.1367996782064438, 0.02590416558086872, -0.004204743541777134, 0.0067570265382528305, -0.00647604139521718, 0.0033333548344671726, -0.002973760711029172, -0.004489073995500803, 0.011359833180904388, -0.01847812533378601, 0.00229805801063776, 0.009392935782670975, -0.0031042180489748716, -0.003917068243026733, 0.012597505934536457, 0.003239693120121956, 0.018518267199397087, -0.025890784338116646, -0.01050349697470665, -0.01364116556942463, -0.01450419146567583, -0.005224987864494324, 0.021716147661209106, 0.032621052116155624, -0.024459097534418106, -0.016216862946748734, 0.03139006718993187, 0.026345713064074516, 0.012477084062993526, 0.0037263997364789248, -0.0028583561070263386, 0.015039402060210705, 0.022666145116090775, -0.0005222479230724275, -0.022947130724787712, -0.002712845802307129, 0.01744784601032734, -0.03390555456280708, 0.023268256336450577, 0.016457708552479744, 0.013159476220607758, 1.6189580492209643e-05, 0.004529214929789305, -0.011319692246615887, 0.029918240383267403, -0.0160161592066288, 0.009118640795350075, 0.002346561523154378, 0.010423215106129646, 0.000640997604932636, 0.016818974167108536, -0.006549632642418146, -0.010824622586369514, -0.012905252166092396, 0.002741278847679496, 0.012543984688818455, -9.455028339289129e-05, -0.004994178656488657, -0.019200660288333893, 0.023455578833818436, -0.027028106153011322, -0.03403935581445694, 0.0021926886402070522, -0.016537990421056747, 0.005773578304797411, -0.019776010885834694, -0.005328684579581022, 0.004037490580230951, -0.016872495412826538, -0.013146095909178257, -0.006546287797391415, -0.008436247706413269, -0.038133714348077774, 0.024820365011692047, 0.00935948546975851, 0.011226030066609383, 0.032861895859241486, 0.0016089751152321696, 0.0051447064615786076, -0.001092999242246151, 0.006897519342601299, -0.04228159040212631, -0.0024970893282443285, 0.004937312565743923, -0.0014433945761993527, 0.00633889390155673, -0.018371084704995155, -0.008663712069392204, -0.024392196908593178, 0.03765202686190605, -0.005820408929139376, 0.0017176896799355745, 0.014704895205795765, -0.02146192267537117, -0.0021090619266033173, 0.017956295982003212, 0.02420487254858017, 0.01799643784761429, -0.0027011381462216377, -0.005957556422799826, 0.01594925858080387, -0.029971761628985405, -0.016859116032719612, -0.020378122106194496, -0.006174985785037279, -0.01644432730972767, 0.007693644147366285, 0.025864023715257645, -0.01304574403911829, -0.010817932896316051, -0.00547921285033226, -0.011266171000897884, 0.004137842450290918, 0.017434466630220413, -0.004773404449224472, 0.00222613918595016, 0.01960206776857376, 0.0072989268228411674, 0.0013798383297398686, -0.02389712817966938, 0.005699986591935158, -0.018103478476405144, 0.03588583320379257, 0.02191685140132904, -0.00632551359012723, -0.00889117643237114, 0.006884139031171799, -0.021622484549880028, 0.018371084704995155, 0.004442243371158838, 0.002185998484492302, 0.02055206522345543, -0.012470393441617489, -0.02198375202715397, -0.011299621313810349, -0.016337286680936813, -0.014169685542583466, 0.02179642766714096, 0.009747512638568878, 0.008295755833387375, -0.014450670219957829, -0.011727789416909218, -0.00022014693240635097, -0.04728580638766289, 0.0143837695941329, -0.014637994579970837, 0.006011077668517828, 0.0007083170348778367, -0.01759503036737442, 0.003314957022666931, -0.030373169109225273, -0.007479560095816851, -0.024699943140149117, -0.0012343281414359808, -0.04262947663664818, -0.0215288233011961, -0.0009157109307125211, -0.026613319292664528, 0.037090055644512177, 0.02673374116420746, 0.019749250262975693, 0.0043552713468670845, -0.012383421882987022, 0.012075676582753658, 0.0149457398802042, 0.016551369801163673, -0.006930969655513763, 0.023509100079536438, -0.009847864508628845, 0.04289708286523819, 0.03229992464184761, 0.018183760344982147, -0.028392892330884933, 0.008830965496599674, 0.008650331757962704, 0.020926712080836296, -0.028366131708025932, -0.004455623682588339, -0.011092227883636951, -0.021863330155611038, -0.023790085688233376, -0.009051739238202572, -0.0027095007244497538, -0.003569181775674224, -0.014276727102696896, -6.977591692702845e-05, 0.049212560057640076, -0.01266440749168396, 0.029275989159941673, -0.011968634091317654, 0.021609105169773102, -0.015106302686035633, -0.0043552713468670845, -0.008730613626539707, -0.015119682997465134, -0.020565444603562355, -0.04621538519859314, 0.005332029890269041, 0.03385203331708908, -0.013407011516392231, 0.009018288925290108, -0.001426669186912477, -0.010302793234586716, -0.015199964866042137, 0.013259828090667725, 0.010536947287619114, -0.01866544969379902, -0.01838446408510208, 0.016537990421056747, -0.019896432757377625, -0.00036879314575344324, -0.00933272484689951, -0.027202049270272255, -0.030025282874703407, 0.00122512923553586, 0.0007798177539370954, 0.009974977001547813, 0.010838002897799015, -0.007653503213077784, -0.015079542063176632, -0.027028106153011322, -0.014169685542583466, 0.008897867053747177, -0.005850514397025108, 0.011011946015059948, 0.0075531513430178165, 0.0004306768241804093, -0.013574264012277126, 0.0215288233011961, 0.03363794833421707, 0.004368651658296585, 0.0054557970724999905, 0.027268949896097183, -0.01366792619228363, -0.0006748663727194071, 0.0006573048303835094, 0.00615826016291976, -0.016698552295565605, -0.030453450977802277, 0.024151353165507317, 0.018959814682602882, -0.012744688428938389, -0.017126720398664474, -0.0006928460788913071, -0.01835770346224308, -0.010383074171841145, -0.0149457398802042, 0.02520839124917984, -0.001534547540359199, -0.01095842570066452, -0.020471783354878426, -0.004442243371158838, -0.006348928902298212, 0.0059140706434845924, 0.02243868075311184, -0.015761936083436012, 0.011032016947865486, -0.0016767126508057117, 0.005569529253989458, 0.006445935927331448, -0.014584473334252834, 0.031764715909957886, -0.018250660970807076, 0.003933793865144253, 0.0012794864596799016, 0.02069924771785736, -0.031095702201128006, 0.0007990518352016807, 0.02499430812895298, 0.024258393794298172, 0.0010143902618438005, 0.00458273570984602, -0.012517224065959454, -0.0011389938881620765, -0.01762179099023342, -0.007787305861711502, -0.026506276801228523, -0.020752768963575363, -0.00012627612159121782, 0.0017494678031653166, 0.007854207418859005, -0.0022612623870372772, -0.03465484827756882, -0.0013238085666671395, -0.0033216471783816814, -0.019802771508693695, -0.016297144815325737, -0.018745731562376022, 0.025248533114790916, -0.0012878491543233395, -0.017795734107494354, -0.021635865792632103, -0.0015337112126871943, 0.027483034878969193, -0.0042013986967504025, 0.00977427326142788, 0.021274598315358162, 0.00273960642516613, -0.027014726772904396, 0.015320386737585068, -0.0386689230799675, 0.00762674305588007, -0.035484425723552704, -0.0057401275262236595, -0.03294217586517334, -0.013955601491034031, 0.009426387026906013, -0.002933620009571314, -0.025890784338116646, -0.020926712080836296, -0.007640122901648283, 0.0016733675729483366, 0.0017084907740354538, -0.01723376289010048, -0.0061984010972082615, -0.002950345166027546, -0.002466983627527952, -0.013848558999598026, -0.02003023587167263, 0.051835089921951294, -0.016671791672706604, -0.005917415954172611, 0.0006552141276188195, -0.024940786883234978, -0.009928146377205849, 0.0008554997621104121, 0.010710890404880047, 0.01277144905179739, -0.009700682014226913, -0.022478820756077766, -0.01050349697470665, 0.02286684885621071, -0.03711681440472603, -0.030373169109225273, -0.01482531800866127, 0.0042181238532066345, -0.020351361483335495, 0.004786784760653973, 0.008382727392017841, -0.013132715597748756, 0.019776010885834694, 0.013052434660494328, 0.014637994579970837, -0.012764759361743927, -0.009807723574340343, -0.0160429198294878, 0.004328511189669371, -0.015347147360444069, -0.01336687058210373, -0.03315626084804535, -0.018036577850580215, 0.027161909267306328, -0.00018878697301261127, -0.014624614268541336, -0.009506667964160442, -0.020177418366074562, -0.023856986314058304, -0.009299274533987045, -0.0320858396589756, 0.0027780744712799788, 0.03136330842971802, 0.02392388880252838, 0.004382031969726086, 0.02401755005121231, -0.011473565362393856, 0.0227062851190567, -0.0045860810205340385, -0.003793301060795784, 0.019200660288333893, -0.03698301315307617, 0.036180198192596436, 0.007533080875873566, -0.02110065519809723, -0.0005523534491658211, 0.008697163313627243, -0.0037197095807641745, 0.02940979041159153, 0.015868976712226868, -0.03631399944424629, 0.0006464333855547011, 0.004669707268476486, 0.005077804904431105, 0.011667578481137753, 0.006666709668934345, 0.01659151166677475, -0.038401320576667786, -0.023669663816690445, -0.0013547503622248769, -0.0020521958358585835, -0.008242234587669373, -0.017648549750447273, 0.012824970297515392, 0.0013606041902676225, 9.967241203412414e-05, -0.015775315463542938, -0.0019786045886576176, 0.03679569065570831, -0.01905347779393196, 0.027750639244914055, 0.015614752657711506, -0.009506667964160442, 0.018678829073905945, 0.027081627398729324, -0.016578130424022675, -0.005091185215860605, -0.005291888955980539, 0.014878838323056698, 0.012597505934536457, -0.003105890704318881, 0.0006731938337907195, 0.03623371943831444, -0.020953472703695297, -0.005248403176665306, -0.012443632818758488, 0.02279994636774063, -0.009847864508628845, -0.005328684579581022, -0.002719535958021879, 0.0074728699401021, 0.007165124174207449, -0.016537990421056747, -0.008509838953614235, 0.01347391214221716, -0.02556965872645378, 0.0032932141330093145, -0.01966896839439869, -0.014865458011627197, -0.010021807625889778, -0.011292931623756886, -0.0027462963480502367, -0.006730265915393829, -0.022813327610492706, -0.0021809807512909174, -0.012918631546199322, -0.00790103804320097, 0.022264737635850906, 0.23185297846794128, 0.0032363480422645807, 0.005178156774491072, 0.01613658294081688, -0.013079195283353329, 0.030373169109225273, 0.010543637908995152, 0.011694339103996754, -0.0033249922562390566, 0.02110065519809723, -0.013875319622457027, -0.011159129440784454, -0.0215288233011961, 0.004137842450290918, 0.00041813284042291343, -0.008743993937969208, -0.03561822697520256, -0.02334853820502758, -0.007834136486053467, 0.0025706805754452944, -0.019401364028453827, 0.009426387026906013, -0.0059140706434845924, -0.014330248348414898, 0.01589573733508587, 0.01613658294081688, 0.007700334303081036, 0.015146443620324135, 0.006770406849682331, -0.0026058037765324116, -0.02277318574488163, -0.0064827315509319305, 0.012657716870307922, -0.0006828109035268426, -0.002025435445830226, -0.011747860349714756, -0.024980928748846054, -0.012564055621623993, -0.0054557970724999905, 0.015695033594965935, -0.020525304600596428, 0.0320858396589756, 0.0016850753454491496, 0.007312307134270668, -0.004555975552648306, 0.041452016681432724, -0.006733611226081848, -0.013025674037635326, 0.008576740510761738, 0.005950866267085075, -0.03157738968729973, 0.011741169728338718, 0.017755592241883278, 0.025261912494897842, 0.0005209934897720814, -0.015587992034852505, 0.0004507472040131688, 0.0032196228858083487, -0.01845136471092701, -0.013848558999598026, -0.015614752657711506, 0.008650331757962704, -0.013594334945082664, 0.0398196242749691, -0.01394222117960453, 0.04046187922358513, 0.021114034578204155, 0.009252442978322506, 0.007606672588735819, -0.013079195283353329, -0.029918240383267403, 0.011573917232453823, -0.011540465988218784, -0.00473326351493597, -0.01750136725604534, -0.01744784601032734, 0.01266440749168396, 0.011266171000897884, 0.013955601491034031, -0.002592423465102911, 0.0037096745800226927, -0.017073199152946472, 0.010858073830604553, 0.004428863059729338, -0.0026593247894197702, -0.03510977700352669, 0.022545721381902695, -0.0034487596713006496, -0.007479560095816851, 0.004699813202023506, 0.003294886788353324, -0.016069680452346802, -0.014852078631520271, -0.008396107703447342, -0.0015220035566017032, -0.009600330144166946, 0.010389764793217182, 0.03524358198046684, -0.01990981213748455, 0.018464745953679085, -0.03125626593828201, 0.021635865792632103, 0.03339710459113121, -0.011125678196549416, -0.005057734902948141, -0.0010428233072161674, -0.0014375406317412853, 0.03957878053188324, 0.02088657207787037, -0.02618515118956566, -0.00048043462447822094, -0.021622484549880028, -0.0043853772804141045, -0.006864068564027548, -0.0021558927837759256, 0.01817038096487522, 0.023121073842048645, -0.021689387038350105, 0.025462616235017776, 0.0028315954841673374, -0.004525869619101286, -0.035725269466638565, -0.031497109681367874, -0.0021124070044606924, 0.0019200659589841962, -0.02012389712035656, -0.007426039315760136, -0.005408966448158026, -0.003910378087311983, -0.02511473000049591, 0.005944176111370325, -0.01105208694934845, 0.033718232065439224, -0.0035825620871037245, -0.0017227072967216372, -0.0007141708629205823, 0.021956991404294968, 0.000201958158868365, -0.01161405723541975, 0.0020521958358585835, -0.005245058331638575, 0.0005021775141358376, 0.005730092525482178, -0.013507363386452198, 0.008590120822191238, -0.01482531800866127, 0.025074589997529984, -0.025489376857876778, -0.0014233241090551019, -0.0035089708399027586, -0.03896328806877136, 0.02645275555551052, 0.008770754560828209, -0.018839392811059952, 0.016818974167108536, -0.029302749782800674, -0.023843606933951378, -0.03561822697520256, -0.01689925603568554, 0.01537390798330307, -0.038561880588531494, 0.02478022500872612, 0.02334853820502758, 0.004770059138536453, -0.019254181534051895, -0.009105260483920574, -0.17083904147148132, 0.03345062583684921, 0.004325165878981352, -0.006877448875457048, 0.00022370106307789683, -0.007211955264210701, 0.009198922663927078, -0.014771796762943268, -0.0009148746612481773, -0.0022712976206094027, 0.017728831619024277, 0.005860549863427877, -0.016992919147014618, -0.011139058507978916, 0.0006305443239398301, -0.018826013430953026, -0.022505581378936768, 0.010449975728988647, 0.022518962621688843, 0.010383074171841145, 0.0474998913705349, -0.026773881167173386, 0.011547156609594822, -0.043699897825717926, -0.0016549697611480951, -0.014999261125922203, -0.00951335858553648, 0.019923193380236626, -0.012717927806079388, -0.020177418366074562, -0.020565444603562355, -0.010021807625889778, 0.006563012953847647, 0.02368304319679737, 0.006111429538577795, 0.0027513140812516212, 0.0023030757438391447, -0.001603957498446107, -0.0013555866898968816, 0.015574611723423004, 0.043887220323085785, 0.020511923357844353, -0.012704548425972462, -0.006549632642418146, -0.015427429229021072, 0.012851730920374393, 0.005730092525482178, -0.00471988320350647, -0.006686780136078596, -0.0032212953083217144, -0.016096441075205803, -0.04335201159119606, -0.010971805080771446, 0.01933446153998375, 0.017514748498797417, -0.0049440027214586735, -0.03631399944424629, -0.00647604139521718, -0.014183065854012966, -0.009968286380171776, -0.006763716693967581, 0.013875319622457027, 0.004987488500773907, 0.001953516621142626, -0.01582883670926094, -0.008182023651897907, -0.023642903193831444, -0.007285546511411667, -0.03382527455687523, 0.01018237043172121, -0.032835133373737335, 0.006322168279439211, -0.007559841498732567, -0.02940979041159153, 0.005061079747974873, 0.01875911094248295, -0.004305095411837101, 0.0028516659513115883, 0.008295755833387375, 0.0062820278108119965, -0.017942916601896286, 0.022960510104894638, -0.01732742413878441, 0.007814066484570503, 0.005532733630388975, 0.011286241002380848, 0.010824622586369514, 0.003038989380002022, -0.00834258645772934, -0.029275989159941673, 0.008603501133620739, -0.03679569065570831, 0.018250660970807076, -0.012048915959894657, -0.008483079262077808, 0.013467222452163696, 0.023616142570972443, -0.014557712711393833, 0.002383357146754861, -0.01966896839439869, 0.001164918066933751, -0.022184455767273903, 0.0031594117172062397, 0.01780911348760128, 0.005653155967593193, 0.016671791672706604, 0.027590077370405197, 0.04337877407670021, 0.013781658373773098, -0.005649810656905174, -0.015320386737585068, 0.011098917573690414, 0.00891793705523014, 0.0037029844243079424, 0.009259133599698544, -0.007218645419925451, -0.010249271988868713, -0.000809923280030489, 0.02143516205251217, -0.028232328593730927, 0.06336886435747147, 0.005395586136728525, -0.023816846311092377, 0.005516008473932743, 0.004549285396933556, -0.012530604377388954, -0.10131525993347168, 0.0042549194768071175, 0.0063355485908687115, 0.014731655828654766, -0.00974082201719284, 0.03168443217873573, 0.003162756795063615, 0.04139849543571472, -0.013072504661977291, 0.032835133373737335, -0.014771796762943268, -0.030159084126353264, -0.007599982433021069, 0.013728137128055096, 0.01887953281402588, 0.0022679525427520275, 0.0011808071285486221, -0.0044857291504740715, -0.005275163799524307, 0.026385854929685593, -0.004435553215444088, 0.003010556334629655, 0.008951387368142605, -0.0138084189966321, -0.013085884973406792, -0.014798557385802269, -0.02115417644381523, 0.01637742668390274, -0.009031669236719608, 0.013661235570907593, -0.005285198800265789, 0.015079542063176632, 0.01714010164141655, 0.006261957343667746, -0.02110065519809723, 0.0037197095807641745, -0.03342386707663536, -0.02203727327287197, 0.02694782428443432, -0.015534470789134502, 0.004833615384995937, -0.020511923357844353, -0.0028065075166523457, -0.0287407785654068, -0.007359137758612633, -0.002222794108092785, -0.006148225162178278, 0.005221643019467592, 0.02219783514738083, -0.016765454784035683, -0.024258393794298172, 0.00988131482154131, -0.0013965636026114225, -0.030881619080901146, 0.004228159319609404, 0.005208262708038092, -0.001064566196873784, -0.014637994579970837, -0.004994178656488657, 0.002246209653094411, 0.00027366794529370964, 0.010831313207745552, -0.010409834794700146, 0.015962639823555946, -0.001418306608684361, -0.02490064688026905, -0.04774073511362076, 0.00243520550429821, 0.008248924277722836, -0.012530604377388954, -0.002197706140577793, 0.013647855259478092, -0.02279994636774063, 0.008851035498082638, -0.0287942998111248, 0.008242234587669373, -0.03786610811948776, -0.0012560710310935974, 0.02036474086344242, -0.027536556124687195, -0.000308372953440994, -0.01744784601032734, 0.01659151166677475, -0.007312307134270668, 0.02542247623205185, 0.009814414195716381, -0.005395586136728525, 0.011821451596915722, 0.0014400494983419776, -0.012557365000247955, 0.008168643340468407, 0.03181823715567589, -0.003017246490344405, -0.009473217651247978, 0.004566010553389788, 0.03275485336780548, 0.008590120822191238, 0.0022127588745206594, 0.008376036770641804, 0.007907727733254433, -0.006258612032979727, 0.0018615273293107748, -0.06352943181991577, 0.022117555141448975, 0.0008157771662808955, 0.0013614405179396272, -0.006031148135662079, 0.014731655828654766, -0.006191710941493511, 0.0004829434328712523, 0.020712627097964287, 0.013520743697881699, -0.006787132006138563, 0.020257700234651566, 0.0221576951444149, -0.012256309390068054, -0.03679569065570831, 0.021609105169773102, 0.014142924919724464, 0.005348755046725273, 0.03743794187903404, 0.005750162526965141, 0.00531195942312479, 0.029356269165873528, 0.014972500503063202, 0.009560189209878445, -0.0004327674687374383, -0.004435553215444088, -0.017434466630220413, 0.005970936734229326, 0.011460185050964355, -0.007834136486053467, -0.0016984555404633284, -0.01392884086817503, -0.011393283493816853, 0.018156999722123146, -0.00719188479706645, -0.027697117999196053, 0.009506667964160442, -0.003893652930855751, 0.00476336944848299, 0.002620856510475278, -0.03141682967543602, -0.024151353165507317, -0.01689925603568554, -0.017394326627254486, -0.004228159319609404, -0.005950866267085075, -0.0057769231498241425, -0.013968981802463531, 0.0012493808753788471, -0.004087666515260935, -0.011072156950831413, 0.0006037838174961507, -0.03189851716160774, 0.003773230593651533, -0.010771101340651512, -0.014584473334252834, 0.004221469163894653, 0.014276727102696896, 0.0032681261654943228, -0.027590077370405197, 0.033959075808525085, 0.01015560980886221, 0.0022729700431227684, -0.01015560980886221, 0.028392892330884933, -0.015815455466508865, -0.02788444235920906, 0.007011251524090767, 0.02623867057263851, -0.025462616235017776, -0.035270340740680695, -0.011861592531204224, -0.001787935965694487, 0.02081966958940029, -0.008663712069392204, -0.007593292277306318, 0.012584125623106956, 0.0034203266259282827, 0.011313001625239849, 0.021823188289999962, 0.021997131407260895, -0.0036260478664189577, -0.012115816585719585, 0.03631399944424629, -0.0060445284470915794, 0.015119682997465134, -0.004790129605680704, 0.020190797746181488, 0.011159129440784454, 0.0020505234133452177, -0.026171769946813583, -0.002947000088170171, -0.0071316733956336975, 0.000536464445758611, -0.011038706637918949, 0.005743472371250391, -0.004790129605680704, 0.0003221713413950056, 0.02664007991552353, 4.006235030828975e-05, -0.005332029890269041, -0.01823728159070015, -0.002037143101915717, 0.00974082201719284, -0.0028951517306268215, 0.02597106620669365, -0.02831261046230793, 0.002057213569059968, -0.02475346438586712, 0.02356262132525444, 0.021301358938217163, 0.023147834464907646, 0.009272513911128044, 0.02809852547943592, -0.003826751606538892, 0.0221576951444149, 0.005362135358154774, -0.02112741582095623, -0.006412485148757696, 0.04551961272954941, 0.02678726240992546, -0.011199269443750381, 0.0064359004609286785, -0.011239410378038883, 0.026252051815390587, 0.008389417082071304, -0.001522839767858386, -0.0014099439140409231, 0.006542942486703396, 0.0040174201130867004, 0.040970329195261, -0.019347842782735825, -0.043887220323085785, 0.012022155337035656, -0.010864763520658016, 0.008610190823674202, 0.011118988506495953, 0.025409096851944923, 0.0008207947248592973, 0.04798157885670662, 0.017822494730353355, 0.0018866152968257666, -0.006653329823166132, -0.010650679469108582, 0.019254181534051895, -0.009111951105296612, 0.004833615384995937, 0.00016683500143699348, -0.013407011516392231, 0.0342802032828331, -0.020070375874638557, 0.003907033242285252, -0.0006200909847393632, -0.027161909267306328, 0.013052434660494328, -0.004238194320350885, 0.016818974167108536, -0.002022090367972851, 0.011874972842633724, 0.002555627841502428, 0.00323802069760859, 0.020043615251779556, 0.018585167825222015, -0.007506320718675852, -0.004512489307671785, 0.012724618427455425, -0.02057882584631443, 0.00280483509413898, -0.01526686642318964, 0.01278482936322689, 0.008516529574990273, -0.022452060133218765, -0.024793604388833046, 0.030025282874703407, -0.018959814682602882, 0.022853467613458633, 0.017421085387468338, 0.018371084704995155, 0.00790103804320097, 0.0031527215614914894, -0.00488713663071394, -0.020164037123322487, -0.037063293159008026, 0.026626698672771454, 0.013534123077988625, -0.008663712069392204, -0.021970370784401894, 0.004602806176990271], 'id': '84c95df5-7fec-4091-a2c5-e2d8e7df2a6c', 'metadata': {'source': '../../modules/state_of_the_union.txt'}, '_distance': 0.4147511124610901}\n" + ] + } + ], "source": [ "print(docs[0].metadata)" ] @@ -267,7 +321,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.8" + "version": "3.11.5" } }, "nbformat": 4, diff --git a/docs/docs/modules/data_connection/indexing.ipynb b/docs/docs/modules/data_connection/indexing.ipynb index 922f238840bbc..a7980a6896ab4 100644 --- a/docs/docs/modules/data_connection/indexing.ipynb +++ b/docs/docs/modules/data_connection/indexing.ipynb @@ -60,7 +60,7 @@ " * document addition by id (`add_documents` method with `ids` argument)\n", " * delete by id (`delete` method with `ids` argument)\n", "\n", - "Compatible Vectorstores: `AnalyticDB`, `AstraDB`, `AwaDB`, `Bagel`, `Cassandra`, `Chroma`, `CouchbaseVectorStore`, `DashVector`, `DatabricksVectorSearch`, `DeepLake`, `Dingo`, `ElasticVectorSearch`, `ElasticsearchStore`, `FAISS`, `HanaDB`, `Milvus`, `MyScale`, `OpenSearchVectorSearch`, `PGVector`, `Pinecone`, `Qdrant`, `Redis`, `Rockset`, `ScaNN`, `SupabaseVectorStore`, `SurrealDBStore`, `TimescaleVector`, `Vald`, `VDMS`, `Vearch`, `VespaStore`, `Weaviate`, `ZepVectorStore`, `TencentVectorDB`, `OpenSearchVectorSearch`.\n", + "Compatible Vectorstores: `AnalyticDB`, `AstraDB`, `AwaDB`, `Bagel`, `Cassandra`, `Chroma`, `CouchbaseVectorStore`, `DashVector`, `DatabricksVectorSearch`, `DeepLake`, `Dingo`, `ElasticVectorSearch`, `ElasticsearchStore`, `FAISS`, `HanaDB`, `LanceDB`, `Milvus`, `MyScale`, `OpenSearchVectorSearch`, `PGVector`, `Pinecone`, `Qdrant`, `Redis`, `Rockset`, `ScaNN`, `SupabaseVectorStore`, `SurrealDBStore`, `TimescaleVector`, `Vald`, `VDMS`, `Vearch`, `VespaStore`, `Weaviate`, `ZepVectorStore`, `TencentVectorDB`, `OpenSearchVectorSearch`.\n", " \n", "## Caution\n", "\n", diff --git a/libs/community/langchain_community/vectorstores/lancedb.py b/libs/community/langchain_community/vectorstores/lancedb.py index 414517793ee44..0c3c514567bfa 100644 --- a/libs/community/langchain_community/vectorstores/lancedb.py +++ b/libs/community/langchain_community/vectorstores/lancedb.py @@ -1,6 +1,8 @@ from __future__ import annotations +import os import uuid +import warnings from typing import Any, Iterable, List, Optional from langchain_core.documents import Document @@ -22,15 +24,14 @@ class LanceDB(VectorStore): id_key: Key to use for the id in the database. Defaults to ``id``. text_key: Key to use for the text in the database. Defaults to ``text``. table_name: Name of the table to use. Defaults to ``vectorstore``. + api_key: API key to use for LanceDB cloud database. + region: Region to use for LanceDB cloud database. Example: .. code-block:: python - - db = lancedb.connect('./lancedb') - table = db.open_table('my_table') - vectorstore = LanceDB(table, embedding_function) + vectorstore = LanceDB(uri='/lancedb', embedding_function) vectorstore.add_texts(['text1', 'text2']) result = vectorstore.similarity_search('text1') """ @@ -39,10 +40,14 @@ def __init__( self, connection: Optional[Any] = None, embedding: Optional[Embeddings] = None, + uri: Optional[str] = "/tmp/lancedb", vector_key: Optional[str] = "vector", id_key: Optional[str] = "id", text_key: Optional[str] = "text", table_name: Optional[str] = "vectorstore", + api_key: Optional[str] = None, + region: Optional[str] = None, + mode: Optional[str] = "overwrite", ): """Initialize with Lance DB vectorstore""" try: @@ -52,25 +57,41 @@ def __init__( "Could not import lancedb python package. " "Please install it with `pip install lancedb`." ) - self.lancedb = lancedb self._embedding = embedding self._vector_key = vector_key self._id_key = id_key self._text_key = text_key self._table_name = table_name + self.api_key = api_key + self.region = region + self.mode = mode + + if os.getenv("LANCE_API_KEY") is not None: + api_key = os.getenv("LANCE_API_KEY") + + if "db://" in uri and api_key is None: + raise ValueError("API key is required for LanceDB cloud.") if self._embedding is None: - raise ValueError("embedding should be provided") + raise ValueError("embedding object should be provided") - if connection is not None: - if not isinstance(connection, lancedb.db.LanceTable): - raise ValueError( - "connection should be an instance of lancedb.db.LanceTable, ", - f"got {type(connection)}", - ) + if isinstance(connection, lancedb.db.LanceDBConnection): self._connection = connection + elif isinstance(connection, str): + raise ValueError( + "`connection` has to be a lancedb.db.LanceDBConnection object." + ) else: - self._connection = self._init_table() + if api_key is None: + self._connection = lancedb.connect(uri) + else: + if "db://" not in uri: + self._connection = lancedb.connect(uri) + warnings.warn( + "api key provided with local uri. \ + The data will be stored locally" + ) + self._connection = lancedb.connect(uri, api_key=api_key, region=region) @property def embeddings(self) -> Optional[Embeddings]: @@ -88,7 +109,7 @@ def add_texts( Args: texts: Iterable of strings to add to the vectorstore. metadatas: Optional list of metadatas associated with the texts. - ids: Optional list of ids to associate with the texts. + ids: Optional list of ids to associate w ith the texts. Returns: List of ids of the added texts. @@ -99,20 +120,70 @@ def add_texts( embeddings = self._embedding.embed_documents(list(texts)) # type: ignore for idx, text in enumerate(texts): embedding = embeddings[idx] - metadata = metadatas[idx] if metadatas else {} + metadata = metadatas[idx] if metadatas else {"id": ids[idx]} docs.append( { self._vector_key: embedding, self._id_key: ids[idx], self._text_key: text, - **metadata, + "metadata": metadata, } ) - self._connection.add(docs) + + if self._table_name in self._connection.table_names(): + tbl = self._connection.open_table(self._table_name) + if self.mode and self.api_key is None: + tbl.add(docs, mode=self.mode) + else: + tbl.add(docs) + else: + self._connection.create_table(self._table_name, data=docs) return ids + def get_table(self, name: Optional[str] = None) -> Any: + if name is not None: + try: + self._connection.open_table(name) + except Exception: + raise ValueError(f"Table {name} not found in the database") + else: + return self._connection.open_table(self._table_name) + + def create_index( + self, + col_name: Optional[str] = None, + vector_col: Optional[str] = None, + num_partitions: Optional[int] = 256, + num_sub_vectors: Optional[int] = 96, + index_cache_size: Optional[int] = None, + ) -> None: + """ + Create a scalar(for non-vector cols) or a vector index on a table. + Make sure your vector column has enough data before creating an index on it. + + Args: + vector_col: Provide if you want to create index on a vector column. + col_name: Provide if you want to create index on a non-vector column. + metric: Provide the metric to use for vector index. Defaults to 'L2' + choice of metrics: 'L2', 'dot', 'cosine' + + Returns: + None + """ + if vector_col: + self._connection.create_index( + vector_column_name=vector_col, + num_partitions=num_partitions, + num_sub_vectors=num_sub_vectors, + index_cache_size=index_cache_size, + ) + elif col_name: + self._connection.create_scalar_index(col_name) + else: + raise ValueError("Provide either vector_col or col_name") + def similarity_search( - self, query: str, k: int = 4, **kwargs: Any + self, query: str, k: int = 4, name: Optional[str] = None, **kwargs: Any ) -> List[Document]: """Return documents most similar to the query @@ -124,8 +195,9 @@ def similarity_search( List of documents most similar to the query. """ embedding = self._embedding.embed_query(query) # type: ignore + tbl = self.get_table(name) docs = ( - self._connection.search(embedding, vector_column_name=self._vector_key) + tbl.search(embedding, vector_column_name=self._vector_key) .limit(k) .to_arrow() ) @@ -155,32 +227,47 @@ def from_texts( **kwargs: Any, ) -> LanceDB: instance = LanceDB( - connection, - embedding, - vector_key, - id_key, - text_key, + connection=connection, + embedding=embedding, + vector_key=vector_key, + id_key=id_key, + text_key=text_key, ) instance.add_texts(texts, metadatas=metadatas, **kwargs) return instance - def _init_table(self) -> Any: - import pyarrow as pa - - schema = pa.schema( - [ - pa.field( - self._vector_key, - pa.list_( - pa.float32(), - len(self.embeddings.embed_query("test")), # type: ignore - ), - ), - pa.field(self._id_key, pa.string()), - pa.field(self._text_key, pa.string()), - ] - ) - db = self.lancedb.connect("/tmp/lancedb") - tbl = db.create_table(self._table_name, schema=schema, mode="overwrite") - return tbl + def delete( + self, + ids: Optional[List[str]] = None, + delete_all: Optional[bool] = None, + filter: Optional[str] = None, + drop_columns: Optional[List[str]] = None, + name: Optional[str] = None, + **kwargs: Any, + ) -> None: + """ + Allows deleting rows by filtering, by ids or drop columns from the table. + + Args: + filter: Provide a string SQL expression - "{col} {operation} {value}". + ids: Provide list of ids to delete from the table. + drop_columns: Provide list of columns to drop from the table. + delete_all: If True, delete all rows from the table. + """ + tbl = self.get_table(name) + if filter: + tbl.delete(filter) + elif ids: + tbl.delete("id in ('{}')".format(",".join(ids))) + elif drop_columns: + if self.api_key is not None: + raise NotImplementedError( + "Column operations currently not supported in LanceDB Cloud." + ) + else: + tbl.drop_columns(drop_columns) + elif delete_all: + tbl.delete("true") + else: + raise ValueError("Provide either filter, ids, drop_columns or delete_all") diff --git a/libs/community/tests/integration_tests/vectorstores/test_lancedb.py b/libs/community/tests/integration_tests/vectorstores/test_lancedb.py index bde46e800e116..f32db739591b4 100644 --- a/libs/community/tests/integration_tests/vectorstores/test_lancedb.py +++ b/libs/community/tests/integration_tests/vectorstores/test_lancedb.py @@ -9,22 +9,18 @@ def test_lancedb_with_connection() -> None: import lancedb embeddings = FakeEmbeddings() - db = lancedb.connect("/tmp/lancedb") + db = lancedb.connect("/tmp/lancedb_connection") texts = ["text 1", "text 2", "item 3"] - vectors = embeddings.embed_documents(texts) - table = db.create_table( - "my_table", - data=[ - {"vector": vectors[idx], "id": text, "text": text} - for idx, text in enumerate(texts) - ], - mode="overwrite", - ) - store = LanceDB(table, embeddings) + store = LanceDB(connection=db, embedding=embeddings) + store.add_texts(texts) + result = store.similarity_search("text 1") result_texts = [doc.page_content for doc in result] assert "text 1" in result_texts + store.delete(filter="text = 'text 1'") + assert store.get_table().count_rows() == 2 + @pytest.mark.requires("lancedb") def test_lancedb_without_connection() -> None: diff --git a/libs/community/tests/unit_tests/vectorstores/test_indexing_docs.py b/libs/community/tests/unit_tests/vectorstores/test_indexing_docs.py index b5b9c4b78e03b..c0e29eb995173 100644 --- a/libs/community/tests/unit_tests/vectorstores/test_indexing_docs.py +++ b/libs/community/tests/unit_tests/vectorstores/test_indexing_docs.py @@ -67,6 +67,7 @@ def check_compatibility(vector_store: VectorStore) -> bool: "FAISS", "HanaDB", "InMemoryVectorStore", + "LanceDB", "Milvus", "MomentoVectorIndex", "MyScale", From c59ba734b5ab2617ef7c2d36edb86511f879a10c Mon Sep 17 00:00:00 2001 From: raghavdixit99 Date: Thu, 18 Apr 2024 21:26:21 -0400 Subject: [PATCH 2/8] cr changes --- .../langchain_community/vectorstores/lancedb.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libs/community/langchain_community/vectorstores/lancedb.py b/libs/community/langchain_community/vectorstores/lancedb.py index 0c3c514567bfa..1826507119a30 100644 --- a/libs/community/langchain_community/vectorstores/lancedb.py +++ b/libs/community/langchain_community/vectorstores/lancedb.py @@ -26,6 +26,7 @@ class LanceDB(VectorStore): table_name: Name of the table to use. Defaults to ``vectorstore``. api_key: API key to use for LanceDB cloud database. region: Region to use for LanceDB cloud database. + mode: Mode to use for adding data to the table. Defaults to ``overwrite``. @@ -62,14 +63,11 @@ def __init__( self._id_key = id_key self._text_key = text_key self._table_name = table_name - self.api_key = api_key + self.api_key = api_key or os.getenv("LANCE_API_KEY") self.region = region self.mode = mode - if os.getenv("LANCE_API_KEY") is not None: - api_key = os.getenv("LANCE_API_KEY") - - if "db://" in uri and api_key is None: + if "db://" in uri and self.api_key is None: raise ValueError("API key is required for LanceDB cloud.") if self._embedding is None: @@ -82,7 +80,7 @@ def __init__( "`connection` has to be a lancedb.db.LanceDBConnection object." ) else: - if api_key is None: + if self.api_key is None: self._connection = lancedb.connect(uri) else: if "db://" not in uri: @@ -91,7 +89,9 @@ def __init__( "api key provided with local uri. \ The data will be stored locally" ) - self._connection = lancedb.connect(uri, api_key=api_key, region=region) + self._connection = lancedb.connect( + uri, api_key=self.api_key, region=self.region + ) @property def embeddings(self) -> Optional[Embeddings]: @@ -132,7 +132,7 @@ def add_texts( if self._table_name in self._connection.table_names(): tbl = self._connection.open_table(self._table_name) - if self.mode and self.api_key is None: + if self.api_key is None: tbl.add(docs, mode=self.mode) else: tbl.add(docs) From 3507323f649fdb2f25a3da0cca1b4abf3c68fd5c Mon Sep 17 00:00:00 2001 From: raghavdixit99 Date: Sat, 20 Apr 2024 01:12:25 -0400 Subject: [PATCH 3/8] updates --- libs/community/langchain_community/vectorstores/lancedb.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/community/langchain_community/vectorstores/lancedb.py b/libs/community/langchain_community/vectorstores/lancedb.py index 1826507119a30..de788a2aa184a 100644 --- a/libs/community/langchain_community/vectorstores/lancedb.py +++ b/libs/community/langchain_community/vectorstores/lancedb.py @@ -75,9 +75,10 @@ def __init__( if isinstance(connection, lancedb.db.LanceDBConnection): self._connection = connection - elif isinstance(connection, str): + elif isinstance(connection, (str, lancedb.db.LanceTable)): raise ValueError( - "`connection` has to be a lancedb.db.LanceDBConnection object." + "`connection` has to be a lancedb.db.LanceDBConnection object.\ + `lancedb.db.LanceTable` is deprecated." ) else: if self.api_key is None: From 017264845956d6cc4f24d88e971b461120aafd20 Mon Sep 17 00:00:00 2001 From: raghavdixit99 Date: Sat, 20 Apr 2024 01:16:55 -0400 Subject: [PATCH 4/8] ipynb lint fix --- .../integrations/vectorstores/lancedb.ipynb | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/docs/docs/integrations/vectorstores/lancedb.ipynb b/docs/docs/integrations/vectorstores/lancedb.ipynb index 9928fcdbc4347..f1dd46433e3ab 100644 --- a/docs/docs/integrations/vectorstores/lancedb.ipynb +++ b/docs/docs/integrations/vectorstores/lancedb.ipynb @@ -65,7 +65,7 @@ "from langchain.document_loaders import TextLoader\n", "from langchain_text_splitters import CharacterTextSplitter\n", "from langchain_openai import OpenAIEmbeddings\n", - "from langchain.vectorstores import LanceDB\n", + "from langchain.vectorstores import LanceDB\n", "\n", "loader = TextLoader(\"../../modules/state_of_the_union.txt\")\n", "documents = loader.load()\n", @@ -110,28 +110,20 @@ ] }, { - "cell_type": "code", - "execution_count": 8, + "cell_type": "markdown", "id": "f5e1cdfd", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "tbl: LanceTable(connection=LanceDBConnection(/tmp/lancedb), name=\"vectorstore\")\n" - ] - } - ], "source": [ - "# to explore the table, you can load it into a df or save it in a csv file: \n", + "Additionaly, to explore the table you can load it into a df or save it in a csv file: \n", + "```python\n", "tbl = docsearch.get_table()\n", "print(\"tbl:\", tbl)\n", "pd_df = tbl.to_pandas()\n", "# pd_df.to_csv(\"docsearch.csv\", index=False)\n", "\n", "# you can also create a new vector store object using an older connection object:\n", - "vector_store = LanceDB(connection=tbl, embedding=embeddings)" + "vector_store = LanceDB(connection=tbl, embedding=embeddings)\n", + "```" ] }, { From 2cf52db1b4109b6b178bbfe391d1b12ad4249497 Mon Sep 17 00:00:00 2001 From: raghavdixit99 Date: Sat, 20 Apr 2024 01:21:02 -0400 Subject: [PATCH 5/8] nb format --- docs/docs/integrations/vectorstores/lancedb.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/integrations/vectorstores/lancedb.ipynb b/docs/docs/integrations/vectorstores/lancedb.ipynb index f1dd46433e3ab..340169de34a79 100644 --- a/docs/docs/integrations/vectorstores/lancedb.ipynb +++ b/docs/docs/integrations/vectorstores/lancedb.ipynb @@ -63,9 +63,9 @@ "outputs": [], "source": [ "from langchain.document_loaders import TextLoader\n", - "from langchain_text_splitters import CharacterTextSplitter\n", - "from langchain_openai import OpenAIEmbeddings\n", "from langchain.vectorstores import LanceDB\n", + "from langchain_openai import OpenAIEmbeddings\n", + "from langchain_text_splitters import CharacterTextSplitter\n", "\n", "loader = TextLoader(\"../../modules/state_of_the_union.txt\")\n", "documents = loader.load()\n", From 1d5f76e50786901cadfd6a78e38f5d84edb047ce Mon Sep 17 00:00:00 2001 From: raghavdixit99 Date: Sat, 20 Apr 2024 02:14:29 -0400 Subject: [PATCH 6/8] mypy compat changes --- .../vectorstores/lancedb.py | 44 +++++++++++-------- .../vectorstores/test_lancedb.py | 15 ++++++- 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/libs/community/langchain_community/vectorstores/lancedb.py b/libs/community/langchain_community/vectorstores/lancedb.py index de788a2aa184a..1da925863e5d6 100644 --- a/libs/community/langchain_community/vectorstores/lancedb.py +++ b/libs/community/langchain_community/vectorstores/lancedb.py @@ -10,6 +10,17 @@ from langchain_core.vectorstores import VectorStore +def import_lancedb() -> Any: + try: + import lancedb + except ImportError as e: + raise ImportError( + "Could not import pinecone lancedb package. " + "Please install it with `pip install lancedb`." + ) from e + return lancedb + + class LanceDB(VectorStore): """`LanceDB` vector store. @@ -51,13 +62,7 @@ def __init__( mode: Optional[str] = "overwrite", ): """Initialize with Lance DB vectorstore""" - try: - import lancedb - except ImportError: - raise ImportError( - "Could not import lancedb python package. " - "Please install it with `pip install lancedb`." - ) + lancedb = import_lancedb() self._embedding = embedding self._vector_key = vector_key self._id_key = id_key @@ -67,8 +72,9 @@ def __init__( self.region = region self.mode = mode - if "db://" in uri and self.api_key is None: - raise ValueError("API key is required for LanceDB cloud.") + if isinstance(uri, str) and self.api_key is None: + if uri.startswith("db://"): + raise ValueError("API key is required for LanceDB cloud.") if self._embedding is None: raise ValueError("embedding object should be provided") @@ -84,15 +90,17 @@ def __init__( if self.api_key is None: self._connection = lancedb.connect(uri) else: - if "db://" not in uri: - self._connection = lancedb.connect(uri) - warnings.warn( - "api key provided with local uri. \ - The data will be stored locally" - ) - self._connection = lancedb.connect( - uri, api_key=self.api_key, region=self.region - ) + if isinstance(uri, str): + if uri.startswith("db://"): + self._connection = lancedb.connect( + uri, api_key=self.api_key, region=self.region + ) + else: + self._connection = lancedb.connect(uri) + warnings.warn( + "api key provided with local uri. \ + The data will be stored locally" + ) @property def embeddings(self) -> Optional[Embeddings]: diff --git a/libs/community/tests/integration_tests/vectorstores/test_lancedb.py b/libs/community/tests/integration_tests/vectorstores/test_lancedb.py index f32db739591b4..d9ddf966e5437 100644 --- a/libs/community/tests/integration_tests/vectorstores/test_lancedb.py +++ b/libs/community/tests/integration_tests/vectorstores/test_lancedb.py @@ -1,12 +1,25 @@ +from typing import Any + import pytest from langchain_community.vectorstores import LanceDB from tests.integration_tests.vectorstores.fake_embeddings import FakeEmbeddings +def import_lancedb() -> Any: + try: + import lancedb + except ImportError as e: + raise ImportError( + "Could not import pinecone lancedb package. " + "Please install it with `pip install lancedb`." + ) from e + return lancedb + + @pytest.mark.requires("lancedb") def test_lancedb_with_connection() -> None: - import lancedb + lancedb = import_lancedb() embeddings = FakeEmbeddings() db = lancedb.connect("/tmp/lancedb_connection") From eaa621abe1a255642addc08021f1e8ada3e2fbf8 Mon Sep 17 00:00:00 2001 From: Raghav Dixit <34462078+raghavdixit99@users.noreply.github.com> Date: Sat, 20 Apr 2024 02:22:38 -0400 Subject: [PATCH 7/8] doc_fix --- docs/docs/integrations/vectorstores/lancedb.ipynb | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/docs/docs/integrations/vectorstores/lancedb.ipynb b/docs/docs/integrations/vectorstores/lancedb.ipynb index 340169de34a79..bd67759b9769f 100644 --- a/docs/docs/integrations/vectorstores/lancedb.ipynb +++ b/docs/docs/integrations/vectorstores/lancedb.ipynb @@ -280,18 +280,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "9d749a3f-df17-4a8a-b256-08a3bbc74cb6", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'vector': [-0.005857204552739859, -0.0019384637707844377, -0.0046228766441345215, -0.002721208380535245, -0.0009709044243209064, 0.019776010885834694, 0.017059819772839546, 0.008590120822191238, -0.019227420911192894, -0.016792215406894684, 0.01087145321071148, 0.010985185392200947, -0.0004419664037413895, -0.007513010874390602, -0.00359259732067585, -0.0006225997931323946, 0.03334358334541321, -0.009974977001547813, 0.00763343321159482, -0.030667534098029137, -0.013835178688168526, 0.0036461183335632086, 0.014664754271507263, 0.006733611226081848, 0.0008337568724527955, 0.010536947287619114, 0.03179147467017174, -0.009807723574340343, 0.011038706637918949, -0.018531646579504013, -0.017300663515925407, 0.006057908292859793, -0.03676892817020416, -0.018678829073905945, -0.027804160490632057, -0.006138190161436796, 9.779081301530823e-05, -0.011145749129354954, 0.01878587156534195, -0.013848558999598026, 0.03588583320379257, -0.02578374370932579, 0.002217776607722044, -0.018732350319623947, -0.01890629343688488, 0.018197141587734222, -0.009433076716959476, -0.020511923357844353, -0.02618515118956566, 0.025623179972171783, 0.04444919154047966, 0.0018464745953679085, -0.028901340439915657, -0.024914026260375977, 0.011841521598398685, -0.013754897750914097, -0.013052434660494328, 0.00946652702987194, -0.010978495702147484, -0.01161405723541975, 0.013848558999598026, 0.0013087557163089514, -0.02234501764178276, 0.01692601665854454, -0.00963378045707941, -0.018491506576538086, 0.001245199586264789, 0.021903470158576965, -0.003853512229397893, 0.015721794217824936, 0.05437733978033066, 0.0028633736073970795, 0.02597106620669365, -0.006847343407571316, 0.031523872166872025, -0.01960206776857376, -0.004994178656488657, -0.004813545383512974, -0.0010010099504143, -0.017862634733319283, 0.022478820756077766, -0.012958772480487823, -0.022425299510359764, 0.01635066606104374, 0.013139406219124794, -0.014785177074372768, -0.0023281637113541365, -0.001144847716204822, 0.0027145184576511383, 0.0005193209508433938, 0.013012293726205826, 0.02453937940299511, -0.00315104890614748, 0.0066098435781896114, -0.00980103388428688, 0.0149457398802042, -0.026961205527186394, 0.021716147661209106, 0.007907727733254433, -0.026265431195497513, 0.016752073541283607, -0.020806290209293365, -0.02623867057263851, -0.011165819130837917, -0.028847821056842804, -0.029784437268972397, -0.00617164047434926, 0.002262934809550643, 0.004475693684071302, 0.011045397259294987, -0.006797167472541332, 0.009292583912611008, 0.0015537815634161234, 0.0074996305629611015, -0.011687648482620716, -0.02189009077847004, -0.018009817227721214, -0.025797123089432716, 0.022291498258709908, -0.03912385180592537, 0.018799252808094025, 0.014611233957111835, 0.0021006993483752012, -0.028045006096363068, 0.01972248964011669, -0.01990981213748455, -0.013888699933886528, 0.008242234587669373, 0.0006455971160903573, 0.004000694956630468, -0.011547156609594822, -0.011567226611077785, 0.02365628257393837, -0.008041530847549438, 0.011560536921024323, 0.007981319911777973, -0.020873190835118294, 0.01750136725604534, -0.013687996193766594, -0.01833094283938408, 0.019200660288333893, -0.01310595590621233, -0.025596419349312782, -0.015521090477705002, 0.012577435933053493, 0.021381640806794167, 0.0004883791552856565, 0.005743472371250391, 0.014731655828654766, 0.0159893985837698, 0.015012641437351704, -0.016390807926654816, -0.00020749842224176973, -0.007934488356113434, 0.005034319125115871, -0.002130804816260934, -0.024365436285734177, 0.0018381119007244706, 0.008837655186653137, 0.0027914547827094793, 0.006389069836586714, -0.019802771508693695, -0.009861244820058346, 0.03650132194161415, 0.023268256336450577, 0.016430947929620743, -0.014557712711393833, 0.027054866775870323, -0.0006815565284341574, 0.025502758100628853, 0.019200660288333893, 0.017648549750447273, -0.015240105800330639, 0.02783092111349106, 0.0004156240320298821, 0.007807376328855753, 0.018277421593666077, -0.008623571135103703, -0.027135148644447327, 0.01732742413878441, 0.01814362034201622, 0.0009550153627060354, 0.012149267829954624, 0.04378018155694008, -0.02301403135061264, -0.009433076716959476, 0.01756826974451542, 0.007599982433021069, -0.023388678207993507, -0.007506320718675852, -0.0012585797812789679, 0.020859811455011368, 0.011306311935186386, -0.012383421882987022, -0.6452491879463196, -0.01450419146567583, 0.017394326627254486, -0.011814760975539684, -0.0038668925408273935, 0.010175680741667747, -0.0010461683850735426, 0.01756826974451542, -0.016430947929620743, 0.015494329854846, -0.013400320895016193, 0.012724618427455425, 0.0006355618825182319, -0.02286684885621071, -0.02635909430682659, -0.02809852547943592, -0.013206307776272297, -0.008282375521957874, 0.031095702201128006, -0.006268647499382496, -0.026385854929685593, 0.004224814008921385, -0.005940831266343594, 0.007392588537186384, 0.008248924277722836, 0.015547851100564003, 0.020525304600596428, -0.016096441075205803, -0.015467570163309574, 0.010570397600531578, -0.029971761628985405, 0.03229992464184761, -0.0016432619886472821, 0.005756852682679892, 0.04000695049762726, -0.0024970893282443285, 0.0032865239772945642, 0.022733045741915703, 0.013467222452163696, 0.021729527041316032, -0.012684477493166924, 0.024740083143115044, 0.03963230177760124, -0.0027312436141073704, 0.025489376857876778, -0.005589599721133709, 0.021970370784401894, -0.009981666691601276, 0.0006426701438613236, 0.02401755005121231, -0.005766888149082661, 0.019521785899996758, -0.0006159096374176443, 0.0012109127128496766, -0.016056301072239876, -0.002257917309179902, -0.006870758719742298, -0.021328119561076164, 0.000647269596811384, 0.0023248186334967613, -0.0060612536035478115, 0.008456318639218807, -0.0227062851190567, -0.011045397259294987, -0.019628828391432762, 0.02855345420539379, 0.012477084062993526, 0.011814760975539684, 0.019013335928320885, -0.006593118421733379, 0.014544332399964333, 0.004375341814011335, -0.010316173546016216, -0.035270340740680695, 0.010102089494466782, -0.004121117293834686, 0.026278812438249588, -0.013279899023473263, -0.005756852682679892, 0.01133976224809885, 0.0014149615308269858, -0.005489247851073742, -0.00027095008408650756, -0.0010821277974173427, 0.016270384192466736, 0.008984838612377644, -0.021542202681303024, 0.005803683772683144, 0.003796646138653159, 0.012122507207095623, 0.025382336229085922, 0.0023482339456677437, 0.010965115390717983, -0.01787601411342621, 0.01087145321071148, 0.022505581378936768, -0.023549241945147514, 0.002558972919359803, 0.012176028452813625, -0.03163091093301773, -0.0032681261654943228, -0.011988705024123192, 0.0238034650683403, 0.007158434018492699, 0.016645031049847603, 0.013179547153413296, -0.013848558999598026, -0.010276032611727715, 0.03626047819852829, -0.03842807933688164, -0.015293626114726067, 0.0055929445661604404, -0.018732350319623947, -0.017487987875938416, 0.006559667643159628, -0.016551369801163673, 0.007686953991651535, 0.018892914056777954, -0.0149457398802042, -0.019976714625954628, 0.005071115214377642, -0.00025464288773946464, 0.012958772480487823, -0.03899005055427551, 0.0154809495434165, 0.004780094604939222, 0.006506146863102913, -0.0030222642235457897, -0.0009215647587552667, -0.01061053853482008, 0.0033132846001535654, -0.004298405721783638, 0.02079290896654129, 0.008998218923807144, 0.015186584554612637, -0.004823580384254456, 0.01404926273971796, -0.011025326326489449, 0.016966158524155617, -0.011426733806729317, -0.0029319473542273045, -0.0014124526642262936, -0.0060980492271482944, -0.0042482297867536545, -0.024726703763008118, -0.026278812438249588, -0.0006384888547472656, -0.0026676873676478863, -0.030185844749212265, -0.027563316747546196, 0.010068638250231743, -0.018678829073905945, -0.02167600579559803, -0.006389069836586714, 0.007412659004330635, -0.0011314674047753215, -0.0154809495434165, 0.015601372346282005, -0.004689777735620737, -0.021970370784401894, 0.015788694843649864, 0.003529041074216366, -0.027964724227786064, -0.014437290839850903, 0.009834484197199345, -0.010389764793217182, 0.041023846715688705, 0.013192927464842796, 0.006753681693226099, -0.0276168379932642, 0.0009893022943288088, -0.013888699933886528, -0.0014283417258411646, -0.0009951561223715544, 0.037678785622119904, 0.016698552295565605, -0.028714017942547798, 0.013587644323706627, 0.016551369801163673, 0.011701028794050217, -0.007278856355696917, -0.012891871854662895, -0.025195011869072914, -0.006971110589802265, 0.01847812533378601, 0.016270384192466736, 0.004074286203831434, 0.015052782371640205, -0.024378817528486252, 0.0209802333265543, -0.0007543116225861013, -0.00763343321159482, -0.01823728159070015, 0.009861244820058346, 0.012697857804596424, 0.03026612661778927, 0.008864415809512138, 0.00574681768193841, 0.0033015769440680742, 0.029061904177069664, 0.010878143832087517, 0.011045397259294987, 0.022077413275837898, -0.010222511366009712, -0.018290802836418152, -0.038374558091163635, 0.009794343262910843, -0.031737953424453735, 0.02131473831832409, 0.002231156686320901, -0.005659846123307943, -0.027777399867773056, -0.026318952441215515, -0.03238020837306976, 0.009640470147132874, 0.01859854906797409, -0.009727442637085915, 0.030373169109225273, -0.010202441364526749, -0.016645031049847603, 0.011406663805246353, 0.017059819772839546, 0.01723376289010048, -0.00834258645772934, 0.004947347566485405, 0.0027864372823387384, -0.001956861699000001, 0.01206898596137762, 0.01095842570066452, -0.0014609560603275895, 0.00033220654586330056, 0.018585167825222015, 0.015641512349247932, 0.031523872166872025, -0.00989469513297081, -0.021836569532752037, 0.002165928017348051, -0.0015253485180437565, 0.04182666167616844, 0.004666362423449755, 0.02609148807823658, 0.02015065774321556, 0.016631651669740677, -0.0008906229049898684, 0.025649940595030785, -0.031764715909957886, 0.0443153902888298, 0.009727442637085915, 0.030373169109225273, 0.013875319622457027, 0.0003307430597487837, 0.002821560250595212, -0.0011047068983316422, 0.008416177704930305, 0.01726052351295948, -0.03521681949496269, -0.0022930405102670193, 0.020833050832152367, 0.014758416451513767, 0.02587740495800972, 0.03355766832828522, 0.030399929732084274, 0.016537990421056747, -0.017407706007361412, -0.0074594896286726, 0.0013162821996957064, -0.010302793234586716, 0.007907727733254433, -0.004703158047050238, -0.018103478476405144, 0.0020789564587175846, -0.017956295982003212, 0.010282722301781178, 0.00774047477170825, 0.031523872166872025, 0.02310769259929657, -0.004736608825623989, 0.0149457398802042, 0.0023800120688974857, 0.003452104749158025, -0.00919223204255104, -0.033744990825653076, 0.027228809893131256, 0.02399078942835331, -0.014905598945915699, -0.003629392944276333, -0.010557017289102077, -0.021635865792632103, -0.02119431644678116, 0.019682347774505615, -0.01738094538450241, -0.007640122901648283, -0.008931317366659641, 0.0298647191375494, -0.004920586943626404, -0.016216862946748734, 0.030587252229452133, -0.013861939311027527, 0.01759503036737442, -0.0014300142647698522, 0.005178156774491072, -0.0013689668849110603, -0.022933749482035637, -0.0039438288658857346, 0.008824274875223637, -0.0030925103928893805, 0.012597505934536457, -0.012015464715659618, 0.023549241945147514, -0.018183760344982147, 0.018183760344982147, -0.009760892949998379, -0.013360179960727692, 0.010664059780538082, 0.002741278847679496, -0.0038066813722252846, -0.027349231764674187, -0.014209826476871967, 0.004395412281155586, -0.011982014402747154, -0.0020806288812309504, -0.024191493168473244, -0.04469003528356552, -0.016658412292599678, 0.1367996782064438, 0.02590416558086872, -0.004204743541777134, 0.0067570265382528305, -0.00647604139521718, 0.0033333548344671726, -0.002973760711029172, -0.004489073995500803, 0.011359833180904388, -0.01847812533378601, 0.00229805801063776, 0.009392935782670975, -0.0031042180489748716, -0.003917068243026733, 0.012597505934536457, 0.003239693120121956, 0.018518267199397087, -0.025890784338116646, -0.01050349697470665, -0.01364116556942463, -0.01450419146567583, -0.005224987864494324, 0.021716147661209106, 0.032621052116155624, -0.024459097534418106, -0.016216862946748734, 0.03139006718993187, 0.026345713064074516, 0.012477084062993526, 0.0037263997364789248, -0.0028583561070263386, 0.015039402060210705, 0.022666145116090775, -0.0005222479230724275, -0.022947130724787712, -0.002712845802307129, 0.01744784601032734, -0.03390555456280708, 0.023268256336450577, 0.016457708552479744, 0.013159476220607758, 1.6189580492209643e-05, 0.004529214929789305, -0.011319692246615887, 0.029918240383267403, -0.0160161592066288, 0.009118640795350075, 0.002346561523154378, 0.010423215106129646, 0.000640997604932636, 0.016818974167108536, -0.006549632642418146, -0.010824622586369514, -0.012905252166092396, 0.002741278847679496, 0.012543984688818455, -9.455028339289129e-05, -0.004994178656488657, -0.019200660288333893, 0.023455578833818436, -0.027028106153011322, -0.03403935581445694, 0.0021926886402070522, -0.016537990421056747, 0.005773578304797411, -0.019776010885834694, -0.005328684579581022, 0.004037490580230951, -0.016872495412826538, -0.013146095909178257, -0.006546287797391415, -0.008436247706413269, -0.038133714348077774, 0.024820365011692047, 0.00935948546975851, 0.011226030066609383, 0.032861895859241486, 0.0016089751152321696, 0.0051447064615786076, -0.001092999242246151, 0.006897519342601299, -0.04228159040212631, -0.0024970893282443285, 0.004937312565743923, -0.0014433945761993527, 0.00633889390155673, -0.018371084704995155, -0.008663712069392204, -0.024392196908593178, 0.03765202686190605, -0.005820408929139376, 0.0017176896799355745, 0.014704895205795765, -0.02146192267537117, -0.0021090619266033173, 0.017956295982003212, 0.02420487254858017, 0.01799643784761429, -0.0027011381462216377, -0.005957556422799826, 0.01594925858080387, -0.029971761628985405, -0.016859116032719612, -0.020378122106194496, -0.006174985785037279, -0.01644432730972767, 0.007693644147366285, 0.025864023715257645, -0.01304574403911829, -0.010817932896316051, -0.00547921285033226, -0.011266171000897884, 0.004137842450290918, 0.017434466630220413, -0.004773404449224472, 0.00222613918595016, 0.01960206776857376, 0.0072989268228411674, 0.0013798383297398686, -0.02389712817966938, 0.005699986591935158, -0.018103478476405144, 0.03588583320379257, 0.02191685140132904, -0.00632551359012723, -0.00889117643237114, 0.006884139031171799, -0.021622484549880028, 0.018371084704995155, 0.004442243371158838, 0.002185998484492302, 0.02055206522345543, -0.012470393441617489, -0.02198375202715397, -0.011299621313810349, -0.016337286680936813, -0.014169685542583466, 0.02179642766714096, 0.009747512638568878, 0.008295755833387375, -0.014450670219957829, -0.011727789416909218, -0.00022014693240635097, -0.04728580638766289, 0.0143837695941329, -0.014637994579970837, 0.006011077668517828, 0.0007083170348778367, -0.01759503036737442, 0.003314957022666931, -0.030373169109225273, -0.007479560095816851, -0.024699943140149117, -0.0012343281414359808, -0.04262947663664818, -0.0215288233011961, -0.0009157109307125211, -0.026613319292664528, 0.037090055644512177, 0.02673374116420746, 0.019749250262975693, 0.0043552713468670845, -0.012383421882987022, 0.012075676582753658, 0.0149457398802042, 0.016551369801163673, -0.006930969655513763, 0.023509100079536438, -0.009847864508628845, 0.04289708286523819, 0.03229992464184761, 0.018183760344982147, -0.028392892330884933, 0.008830965496599674, 0.008650331757962704, 0.020926712080836296, -0.028366131708025932, -0.004455623682588339, -0.011092227883636951, -0.021863330155611038, -0.023790085688233376, -0.009051739238202572, -0.0027095007244497538, -0.003569181775674224, -0.014276727102696896, -6.977591692702845e-05, 0.049212560057640076, -0.01266440749168396, 0.029275989159941673, -0.011968634091317654, 0.021609105169773102, -0.015106302686035633, -0.0043552713468670845, -0.008730613626539707, -0.015119682997465134, -0.020565444603562355, -0.04621538519859314, 0.005332029890269041, 0.03385203331708908, -0.013407011516392231, 0.009018288925290108, -0.001426669186912477, -0.010302793234586716, -0.015199964866042137, 0.013259828090667725, 0.010536947287619114, -0.01866544969379902, -0.01838446408510208, 0.016537990421056747, -0.019896432757377625, -0.00036879314575344324, -0.00933272484689951, -0.027202049270272255, -0.030025282874703407, 0.00122512923553586, 0.0007798177539370954, 0.009974977001547813, 0.010838002897799015, -0.007653503213077784, -0.015079542063176632, -0.027028106153011322, -0.014169685542583466, 0.008897867053747177, -0.005850514397025108, 0.011011946015059948, 0.0075531513430178165, 0.0004306768241804093, -0.013574264012277126, 0.0215288233011961, 0.03363794833421707, 0.004368651658296585, 0.0054557970724999905, 0.027268949896097183, -0.01366792619228363, -0.0006748663727194071, 0.0006573048303835094, 0.00615826016291976, -0.016698552295565605, -0.030453450977802277, 0.024151353165507317, 0.018959814682602882, -0.012744688428938389, -0.017126720398664474, -0.0006928460788913071, -0.01835770346224308, -0.010383074171841145, -0.0149457398802042, 0.02520839124917984, -0.001534547540359199, -0.01095842570066452, -0.020471783354878426, -0.004442243371158838, -0.006348928902298212, 0.0059140706434845924, 0.02243868075311184, -0.015761936083436012, 0.011032016947865486, -0.0016767126508057117, 0.005569529253989458, 0.006445935927331448, -0.014584473334252834, 0.031764715909957886, -0.018250660970807076, 0.003933793865144253, 0.0012794864596799016, 0.02069924771785736, -0.031095702201128006, 0.0007990518352016807, 0.02499430812895298, 0.024258393794298172, 0.0010143902618438005, 0.00458273570984602, -0.012517224065959454, -0.0011389938881620765, -0.01762179099023342, -0.007787305861711502, -0.026506276801228523, -0.020752768963575363, -0.00012627612159121782, 0.0017494678031653166, 0.007854207418859005, -0.0022612623870372772, -0.03465484827756882, -0.0013238085666671395, -0.0033216471783816814, -0.019802771508693695, -0.016297144815325737, -0.018745731562376022, 0.025248533114790916, -0.0012878491543233395, -0.017795734107494354, -0.021635865792632103, -0.0015337112126871943, 0.027483034878969193, -0.0042013986967504025, 0.00977427326142788, 0.021274598315358162, 0.00273960642516613, -0.027014726772904396, 0.015320386737585068, -0.0386689230799675, 0.00762674305588007, -0.035484425723552704, -0.0057401275262236595, -0.03294217586517334, -0.013955601491034031, 0.009426387026906013, -0.002933620009571314, -0.025890784338116646, -0.020926712080836296, -0.007640122901648283, 0.0016733675729483366, 0.0017084907740354538, -0.01723376289010048, -0.0061984010972082615, -0.002950345166027546, -0.002466983627527952, -0.013848558999598026, -0.02003023587167263, 0.051835089921951294, -0.016671791672706604, -0.005917415954172611, 0.0006552141276188195, -0.024940786883234978, -0.009928146377205849, 0.0008554997621104121, 0.010710890404880047, 0.01277144905179739, -0.009700682014226913, -0.022478820756077766, -0.01050349697470665, 0.02286684885621071, -0.03711681440472603, -0.030373169109225273, -0.01482531800866127, 0.0042181238532066345, -0.020351361483335495, 0.004786784760653973, 0.008382727392017841, -0.013132715597748756, 0.019776010885834694, 0.013052434660494328, 0.014637994579970837, -0.012764759361743927, -0.009807723574340343, -0.0160429198294878, 0.004328511189669371, -0.015347147360444069, -0.01336687058210373, -0.03315626084804535, -0.018036577850580215, 0.027161909267306328, -0.00018878697301261127, -0.014624614268541336, -0.009506667964160442, -0.020177418366074562, -0.023856986314058304, -0.009299274533987045, -0.0320858396589756, 0.0027780744712799788, 0.03136330842971802, 0.02392388880252838, 0.004382031969726086, 0.02401755005121231, -0.011473565362393856, 0.0227062851190567, -0.0045860810205340385, -0.003793301060795784, 0.019200660288333893, -0.03698301315307617, 0.036180198192596436, 0.007533080875873566, -0.02110065519809723, -0.0005523534491658211, 0.008697163313627243, -0.0037197095807641745, 0.02940979041159153, 0.015868976712226868, -0.03631399944424629, 0.0006464333855547011, 0.004669707268476486, 0.005077804904431105, 0.011667578481137753, 0.006666709668934345, 0.01659151166677475, -0.038401320576667786, -0.023669663816690445, -0.0013547503622248769, -0.0020521958358585835, -0.008242234587669373, -0.017648549750447273, 0.012824970297515392, 0.0013606041902676225, 9.967241203412414e-05, -0.015775315463542938, -0.0019786045886576176, 0.03679569065570831, -0.01905347779393196, 0.027750639244914055, 0.015614752657711506, -0.009506667964160442, 0.018678829073905945, 0.027081627398729324, -0.016578130424022675, -0.005091185215860605, -0.005291888955980539, 0.014878838323056698, 0.012597505934536457, -0.003105890704318881, 0.0006731938337907195, 0.03623371943831444, -0.020953472703695297, -0.005248403176665306, -0.012443632818758488, 0.02279994636774063, -0.009847864508628845, -0.005328684579581022, -0.002719535958021879, 0.0074728699401021, 0.007165124174207449, -0.016537990421056747, -0.008509838953614235, 0.01347391214221716, -0.02556965872645378, 0.0032932141330093145, -0.01966896839439869, -0.014865458011627197, -0.010021807625889778, -0.011292931623756886, -0.0027462963480502367, -0.006730265915393829, -0.022813327610492706, -0.0021809807512909174, -0.012918631546199322, -0.00790103804320097, 0.022264737635850906, 0.23185297846794128, 0.0032363480422645807, 0.005178156774491072, 0.01613658294081688, -0.013079195283353329, 0.030373169109225273, 0.010543637908995152, 0.011694339103996754, -0.0033249922562390566, 0.02110065519809723, -0.013875319622457027, -0.011159129440784454, -0.0215288233011961, 0.004137842450290918, 0.00041813284042291343, -0.008743993937969208, -0.03561822697520256, -0.02334853820502758, -0.007834136486053467, 0.0025706805754452944, -0.019401364028453827, 0.009426387026906013, -0.0059140706434845924, -0.014330248348414898, 0.01589573733508587, 0.01613658294081688, 0.007700334303081036, 0.015146443620324135, 0.006770406849682331, -0.0026058037765324116, -0.02277318574488163, -0.0064827315509319305, 0.012657716870307922, -0.0006828109035268426, -0.002025435445830226, -0.011747860349714756, -0.024980928748846054, -0.012564055621623993, -0.0054557970724999905, 0.015695033594965935, -0.020525304600596428, 0.0320858396589756, 0.0016850753454491496, 0.007312307134270668, -0.004555975552648306, 0.041452016681432724, -0.006733611226081848, -0.013025674037635326, 0.008576740510761738, 0.005950866267085075, -0.03157738968729973, 0.011741169728338718, 0.017755592241883278, 0.025261912494897842, 0.0005209934897720814, -0.015587992034852505, 0.0004507472040131688, 0.0032196228858083487, -0.01845136471092701, -0.013848558999598026, -0.015614752657711506, 0.008650331757962704, -0.013594334945082664, 0.0398196242749691, -0.01394222117960453, 0.04046187922358513, 0.021114034578204155, 0.009252442978322506, 0.007606672588735819, -0.013079195283353329, -0.029918240383267403, 0.011573917232453823, -0.011540465988218784, -0.00473326351493597, -0.01750136725604534, -0.01744784601032734, 0.01266440749168396, 0.011266171000897884, 0.013955601491034031, -0.002592423465102911, 0.0037096745800226927, -0.017073199152946472, 0.010858073830604553, 0.004428863059729338, -0.0026593247894197702, -0.03510977700352669, 0.022545721381902695, -0.0034487596713006496, -0.007479560095816851, 0.004699813202023506, 0.003294886788353324, -0.016069680452346802, -0.014852078631520271, -0.008396107703447342, -0.0015220035566017032, -0.009600330144166946, 0.010389764793217182, 0.03524358198046684, -0.01990981213748455, 0.018464745953679085, -0.03125626593828201, 0.021635865792632103, 0.03339710459113121, -0.011125678196549416, -0.005057734902948141, -0.0010428233072161674, -0.0014375406317412853, 0.03957878053188324, 0.02088657207787037, -0.02618515118956566, -0.00048043462447822094, -0.021622484549880028, -0.0043853772804141045, -0.006864068564027548, -0.0021558927837759256, 0.01817038096487522, 0.023121073842048645, -0.021689387038350105, 0.025462616235017776, 0.0028315954841673374, -0.004525869619101286, -0.035725269466638565, -0.031497109681367874, -0.0021124070044606924, 0.0019200659589841962, -0.02012389712035656, -0.007426039315760136, -0.005408966448158026, -0.003910378087311983, -0.02511473000049591, 0.005944176111370325, -0.01105208694934845, 0.033718232065439224, -0.0035825620871037245, -0.0017227072967216372, -0.0007141708629205823, 0.021956991404294968, 0.000201958158868365, -0.01161405723541975, 0.0020521958358585835, -0.005245058331638575, 0.0005021775141358376, 0.005730092525482178, -0.013507363386452198, 0.008590120822191238, -0.01482531800866127, 0.025074589997529984, -0.025489376857876778, -0.0014233241090551019, -0.0035089708399027586, -0.03896328806877136, 0.02645275555551052, 0.008770754560828209, -0.018839392811059952, 0.016818974167108536, -0.029302749782800674, -0.023843606933951378, -0.03561822697520256, -0.01689925603568554, 0.01537390798330307, -0.038561880588531494, 0.02478022500872612, 0.02334853820502758, 0.004770059138536453, -0.019254181534051895, -0.009105260483920574, -0.17083904147148132, 0.03345062583684921, 0.004325165878981352, -0.006877448875457048, 0.00022370106307789683, -0.007211955264210701, 0.009198922663927078, -0.014771796762943268, -0.0009148746612481773, -0.0022712976206094027, 0.017728831619024277, 0.005860549863427877, -0.016992919147014618, -0.011139058507978916, 0.0006305443239398301, -0.018826013430953026, -0.022505581378936768, 0.010449975728988647, 0.022518962621688843, 0.010383074171841145, 0.0474998913705349, -0.026773881167173386, 0.011547156609594822, -0.043699897825717926, -0.0016549697611480951, -0.014999261125922203, -0.00951335858553648, 0.019923193380236626, -0.012717927806079388, -0.020177418366074562, -0.020565444603562355, -0.010021807625889778, 0.006563012953847647, 0.02368304319679737, 0.006111429538577795, 0.0027513140812516212, 0.0023030757438391447, -0.001603957498446107, -0.0013555866898968816, 0.015574611723423004, 0.043887220323085785, 0.020511923357844353, -0.012704548425972462, -0.006549632642418146, -0.015427429229021072, 0.012851730920374393, 0.005730092525482178, -0.00471988320350647, -0.006686780136078596, -0.0032212953083217144, -0.016096441075205803, -0.04335201159119606, -0.010971805080771446, 0.01933446153998375, 0.017514748498797417, -0.0049440027214586735, -0.03631399944424629, -0.00647604139521718, -0.014183065854012966, -0.009968286380171776, -0.006763716693967581, 0.013875319622457027, 0.004987488500773907, 0.001953516621142626, -0.01582883670926094, -0.008182023651897907, -0.023642903193831444, -0.007285546511411667, -0.03382527455687523, 0.01018237043172121, -0.032835133373737335, 0.006322168279439211, -0.007559841498732567, -0.02940979041159153, 0.005061079747974873, 0.01875911094248295, -0.004305095411837101, 0.0028516659513115883, 0.008295755833387375, 0.0062820278108119965, -0.017942916601896286, 0.022960510104894638, -0.01732742413878441, 0.007814066484570503, 0.005532733630388975, 0.011286241002380848, 0.010824622586369514, 0.003038989380002022, -0.00834258645772934, -0.029275989159941673, 0.008603501133620739, -0.03679569065570831, 0.018250660970807076, -0.012048915959894657, -0.008483079262077808, 0.013467222452163696, 0.023616142570972443, -0.014557712711393833, 0.002383357146754861, -0.01966896839439869, 0.001164918066933751, -0.022184455767273903, 0.0031594117172062397, 0.01780911348760128, 0.005653155967593193, 0.016671791672706604, 0.027590077370405197, 0.04337877407670021, 0.013781658373773098, -0.005649810656905174, -0.015320386737585068, 0.011098917573690414, 0.00891793705523014, 0.0037029844243079424, 0.009259133599698544, -0.007218645419925451, -0.010249271988868713, -0.000809923280030489, 0.02143516205251217, -0.028232328593730927, 0.06336886435747147, 0.005395586136728525, -0.023816846311092377, 0.005516008473932743, 0.004549285396933556, -0.012530604377388954, -0.10131525993347168, 0.0042549194768071175, 0.0063355485908687115, 0.014731655828654766, -0.00974082201719284, 0.03168443217873573, 0.003162756795063615, 0.04139849543571472, -0.013072504661977291, 0.032835133373737335, -0.014771796762943268, -0.030159084126353264, -0.007599982433021069, 0.013728137128055096, 0.01887953281402588, 0.0022679525427520275, 0.0011808071285486221, -0.0044857291504740715, -0.005275163799524307, 0.026385854929685593, -0.004435553215444088, 0.003010556334629655, 0.008951387368142605, -0.0138084189966321, -0.013085884973406792, -0.014798557385802269, -0.02115417644381523, 0.01637742668390274, -0.009031669236719608, 0.013661235570907593, -0.005285198800265789, 0.015079542063176632, 0.01714010164141655, 0.006261957343667746, -0.02110065519809723, 0.0037197095807641745, -0.03342386707663536, -0.02203727327287197, 0.02694782428443432, -0.015534470789134502, 0.004833615384995937, -0.020511923357844353, -0.0028065075166523457, -0.0287407785654068, -0.007359137758612633, -0.002222794108092785, -0.006148225162178278, 0.005221643019467592, 0.02219783514738083, -0.016765454784035683, -0.024258393794298172, 0.00988131482154131, -0.0013965636026114225, -0.030881619080901146, 0.004228159319609404, 0.005208262708038092, -0.001064566196873784, -0.014637994579970837, -0.004994178656488657, 0.002246209653094411, 0.00027366794529370964, 0.010831313207745552, -0.010409834794700146, 0.015962639823555946, -0.001418306608684361, -0.02490064688026905, -0.04774073511362076, 0.00243520550429821, 0.008248924277722836, -0.012530604377388954, -0.002197706140577793, 0.013647855259478092, -0.02279994636774063, 0.008851035498082638, -0.0287942998111248, 0.008242234587669373, -0.03786610811948776, -0.0012560710310935974, 0.02036474086344242, -0.027536556124687195, -0.000308372953440994, -0.01744784601032734, 0.01659151166677475, -0.007312307134270668, 0.02542247623205185, 0.009814414195716381, -0.005395586136728525, 0.011821451596915722, 0.0014400494983419776, -0.012557365000247955, 0.008168643340468407, 0.03181823715567589, -0.003017246490344405, -0.009473217651247978, 0.004566010553389788, 0.03275485336780548, 0.008590120822191238, 0.0022127588745206594, 0.008376036770641804, 0.007907727733254433, -0.006258612032979727, 0.0018615273293107748, -0.06352943181991577, 0.022117555141448975, 0.0008157771662808955, 0.0013614405179396272, -0.006031148135662079, 0.014731655828654766, -0.006191710941493511, 0.0004829434328712523, 0.020712627097964287, 0.013520743697881699, -0.006787132006138563, 0.020257700234651566, 0.0221576951444149, -0.012256309390068054, -0.03679569065570831, 0.021609105169773102, 0.014142924919724464, 0.005348755046725273, 0.03743794187903404, 0.005750162526965141, 0.00531195942312479, 0.029356269165873528, 0.014972500503063202, 0.009560189209878445, -0.0004327674687374383, -0.004435553215444088, -0.017434466630220413, 0.005970936734229326, 0.011460185050964355, -0.007834136486053467, -0.0016984555404633284, -0.01392884086817503, -0.011393283493816853, 0.018156999722123146, -0.00719188479706645, -0.027697117999196053, 0.009506667964160442, -0.003893652930855751, 0.00476336944848299, 0.002620856510475278, -0.03141682967543602, -0.024151353165507317, -0.01689925603568554, -0.017394326627254486, -0.004228159319609404, -0.005950866267085075, -0.0057769231498241425, -0.013968981802463531, 0.0012493808753788471, -0.004087666515260935, -0.011072156950831413, 0.0006037838174961507, -0.03189851716160774, 0.003773230593651533, -0.010771101340651512, -0.014584473334252834, 0.004221469163894653, 0.014276727102696896, 0.0032681261654943228, -0.027590077370405197, 0.033959075808525085, 0.01015560980886221, 0.0022729700431227684, -0.01015560980886221, 0.028392892330884933, -0.015815455466508865, -0.02788444235920906, 0.007011251524090767, 0.02623867057263851, -0.025462616235017776, -0.035270340740680695, -0.011861592531204224, -0.001787935965694487, 0.02081966958940029, -0.008663712069392204, -0.007593292277306318, 0.012584125623106956, 0.0034203266259282827, 0.011313001625239849, 0.021823188289999962, 0.021997131407260895, -0.0036260478664189577, -0.012115816585719585, 0.03631399944424629, -0.0060445284470915794, 0.015119682997465134, -0.004790129605680704, 0.020190797746181488, 0.011159129440784454, 0.0020505234133452177, -0.026171769946813583, -0.002947000088170171, -0.0071316733956336975, 0.000536464445758611, -0.011038706637918949, 0.005743472371250391, -0.004790129605680704, 0.0003221713413950056, 0.02664007991552353, 4.006235030828975e-05, -0.005332029890269041, -0.01823728159070015, -0.002037143101915717, 0.00974082201719284, -0.0028951517306268215, 0.02597106620669365, -0.02831261046230793, 0.002057213569059968, -0.02475346438586712, 0.02356262132525444, 0.021301358938217163, 0.023147834464907646, 0.009272513911128044, 0.02809852547943592, -0.003826751606538892, 0.0221576951444149, 0.005362135358154774, -0.02112741582095623, -0.006412485148757696, 0.04551961272954941, 0.02678726240992546, -0.011199269443750381, 0.0064359004609286785, -0.011239410378038883, 0.026252051815390587, 0.008389417082071304, -0.001522839767858386, -0.0014099439140409231, 0.006542942486703396, 0.0040174201130867004, 0.040970329195261, -0.019347842782735825, -0.043887220323085785, 0.012022155337035656, -0.010864763520658016, 0.008610190823674202, 0.011118988506495953, 0.025409096851944923, 0.0008207947248592973, 0.04798157885670662, 0.017822494730353355, 0.0018866152968257666, -0.006653329823166132, -0.010650679469108582, 0.019254181534051895, -0.009111951105296612, 0.004833615384995937, 0.00016683500143699348, -0.013407011516392231, 0.0342802032828331, -0.020070375874638557, 0.003907033242285252, -0.0006200909847393632, -0.027161909267306328, 0.013052434660494328, -0.004238194320350885, 0.016818974167108536, -0.002022090367972851, 0.011874972842633724, 0.002555627841502428, 0.00323802069760859, 0.020043615251779556, 0.018585167825222015, -0.007506320718675852, -0.004512489307671785, 0.012724618427455425, -0.02057882584631443, 0.00280483509413898, -0.01526686642318964, 0.01278482936322689, 0.008516529574990273, -0.022452060133218765, -0.024793604388833046, 0.030025282874703407, -0.018959814682602882, 0.022853467613458633, 0.017421085387468338, 0.018371084704995155, 0.00790103804320097, 0.0031527215614914894, -0.00488713663071394, -0.020164037123322487, -0.037063293159008026, 0.026626698672771454, 0.013534123077988625, -0.008663712069392204, -0.021970370784401894, 0.004602806176990271], 'id': '84c95df5-7fec-4091-a2c5-e2d8e7df2a6c', 'metadata': {'source': '../../modules/state_of_the_union.txt'}, '_distance': 0.4147511124610901}\n" - ] - } - ], + "outputs": [], "source": [ "print(docs[0].metadata)" ] From f4158ffeacff30836879aeb3b12bd077c8741135 Mon Sep 17 00:00:00 2001 From: raghavdixit99 Date: Sat, 20 Apr 2024 02:46:11 -0400 Subject: [PATCH 8/8] update --- libs/community/langchain_community/vectorstores/lancedb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/community/langchain_community/vectorstores/lancedb.py b/libs/community/langchain_community/vectorstores/lancedb.py index 1da925863e5d6..e591fc76664c9 100644 --- a/libs/community/langchain_community/vectorstores/lancedb.py +++ b/libs/community/langchain_community/vectorstores/lancedb.py @@ -68,7 +68,7 @@ def __init__( self._id_key = id_key self._text_key = text_key self._table_name = table_name - self.api_key = api_key or os.getenv("LANCE_API_KEY") + self.api_key = api_key or os.getenv("LANCE_API_KEY") if api_key != "" else None self.region = region self.mode = mode @@ -98,7 +98,7 @@ def __init__( else: self._connection = lancedb.connect(uri) warnings.warn( - "api key provided with local uri. \ + "api key provided with local uri.\ The data will be stored locally" )