Skip to content

Commit aa63ff6

Browse files
author
liyan.90210
committed
feat auto update sdk
1 parent b6d17db commit aa63ff6

File tree

7 files changed

+93
-88
lines changed

7 files changed

+93
-88
lines changed

‎Changelog‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
### Change log
22

3+
2024-11-21 Bumped to version v1.0.162
4+
- Updated apis for viking_db
5+
36
2024-11-14 Bumped to version v1.0.161
47
- Updated apis for vod
58

‎volcengine/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
# coding:utf-8
2-
VERSION='v1.0.161'
2+
VERSION='v1.0.162'

‎volcengine/example/viking_db/example.py‎

Lines changed: 24 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,8 @@ def gen_random_vector(dim):
7474
# datas = [data1, data2, data3, data4]
7575
# collection.upsert_data(datas)
7676

77-
# collection = vikingdb_service.get_collection("sparse")
78-
# res = collection.fetch_data("111")
77+
# collection = vikingdb_service.get_collection("example")
78+
# res = collection.fetch_data(555)
7979
# print(res.fields)
8080

8181
# vector_index = VectorIndexParams(distance=DistanceType.COSINE, index_type=IndexType.HNSW_HYBRID,
@@ -85,11 +85,11 @@ def gen_random_vector(dim):
8585
# res = vikingdb_service.get_index("sparse", "sparse")
8686
# print(res.vector_index)
8787

88-
# index = vikingdb_service.get_index("sparse", "sparse")
89-
# res = index.search_by_vector(vector=gen_random_vector(10), sparse_vectors={"he": 0.05}, dense_weight=0.1)
88+
# index = vikingdb_service.get_index("sparse_go", "sparse_go_test5")
89+
# res = index.search_by_vector(vector=gen_random_vector(12), sparse_vectors={"he": 0.05}, dense_weight=0.1, retry=True)
9090
# print(res)
9191

92-
# index = vikingdb_service.get_index("sparse", "sparse")
92+
# index = vikingdb_service.get_index("sparse_go", "sparse_go_test5")
9393
# res = index.search_by_id("111", dense_weight=0.1)
9494
# print(res)
9595

@@ -130,7 +130,7 @@ def gen_random_vector(dim):
130130
# Field(
131131
# field_name="text_vector",
132132
# field_type=FieldType.Vector,
133-
# dim=10
133+
# dim=12
134134
# ),
135135
# Field(
136136
# field_name="like",
@@ -153,7 +153,7 @@ def gen_random_vector(dim):
153153
# default_val=True
154154
# ),
155155
# ]
156-
# res = vikingdb_service.create_collection("example", fields, "This is an example")
156+
# res = vikingdb_service.create_collection("example1", fields, "This is an example")
157157
# # 返回一个collection实例
158158
# print(res)
159159
#
@@ -171,16 +171,16 @@ def gen_random_vector(dim):
171171
#
172172
# vector_index = VectorIndexParams(distance=DistanceType.COSINE,index_type=IndexType.DiskANN,
173173
# quant=QuantType.Float)
174-
# res = vikingdb_service.create_index("example","test2", vector_index, cpu_quota=2,
174+
# res = vikingdb_service.create_index("example1","example1", vector_index, cpu_quota=2,
175175
# description="This is an index", scalar_index=['price', 'like'], shard_policy=ShardType.Custom, shard_count=10)
176176
# # 返回一个index实例
177177
# print(res)
178178
#
179-
# res = vikingdb_service.get_index("example", "test2")
180-
# # 返回一个index实例
179+
# res = vikingdb_service.get_index("example", "example")
180+
# 返回一个index实例
181181
# print(res.shard_count, res.shard_policy)
182182
#
183-
# vikingdb_service.drop_index("example", "example_index") # 无返回
183+
# vikingdb_service.drop_index("example", "example") # 无返回
184184
#
185185
# res = vikingdb_service.list_indexes("example")
186186
# 返回一个列表
@@ -214,32 +214,31 @@ def gen_random_vector(dim):
214214
# datas.append(data3)
215215
# datas.append(data4)
216216
# collection.upsert_data(datas) # 无返回
217+
217218
#
218219
# collection = vikingdb_service.get_collection("example")
219220
# res = collection.fetch_data("111")
220221
# print(res.fields)
221222
# res = collection.fetch_data(["111", "222", "333", "444"])
222-
# 返回一个列表
223+
# # 返回一个列表
223224
# for item in res:
224225
# print(item)
225226
# print(item.fields)
226227
#
227228
# collection = vikingdb_service.get_collection("example")
228229
# collection.delete_data("11") # 无返回
229230
#
230-
# index = vikingdb_service.get_index("example", "example_index")
231-
# res = index.fetch_data(["11", "33"], partition="1", output_fields=["doc_id", "like"])
232-
# 返回一个列表
231+
# index = vikingdb_service.get_index("sparse_go", "sparse_go_test5")
232+
# res = index.fetch_data(["111"])
233+
# # 返回一个列表
233234
# for item in res:
234235
# print(item)
235236
# print(item.fields)
236237
#
237-
# index = vikingdb_service.get_index("example", "example_index")
238-
# res = index.search_by_id("111", limit=2, output_fields=["doc_id", "like", "text_vector"])
239-
# 返回一个列表
240-
# for item in res:
241-
# print(item)
242-
# print(item.score)
238+
# index = vikingdb_service.get_index("sparse_go", "sparse_go_test5")
239+
# res = index.search_by_vector(gen_random_vector(10), sparse_vectors={'1':1.1})
240+
# # 返回一个列表
241+
# print(res)
243242

244243
# index = vikingdb_service.get_index("example", "example_index")
245244
# def gen_random_vector(dim):
@@ -317,11 +316,9 @@ def gen_random_vector(dim):
317316
# ]
318317
# res = vikingdb_service.create_collection("example_text", fields, "This is an example include text")
319318
#
320-
# vector_index = VectorIndexParams(distance=DistanceType.COSINE, index_type=IndexType.HNSW,
321-
# quant=QuantType.Float)
322-
# res = vikingdb_service.create_index("example_text", "example_index_text", vector_index, cpu_quota=2,
323-
# description="This is an index include text", partition_by="like",
324-
# scalar_index=None)
319+
# vector_index = VectorIndexParams(distance=DistanceType.IP, index_type=IndexType.FLAT,
320+
# quant=QuantType.Int8)
321+
# res = vikingdb_service.create_index("example", "example1")
325322
#
326323
# collection = vikingdb_service.get_collection("example_text")
327324
# field1 = {"doc_id": "11", "text": {"text":"this is one"}, "like": 1, "price": 1.11,
@@ -531,7 +528,7 @@ def gen_random_vector(dim):
531528
# res[i] = random.random() - 0.5
532529
# return res
533530

534-
# collection = await vikingdb_service.async_get_collection("async")
531+
# collection = await vikingdb_service.async_get_collection("example")
535532
# field1 = {"doc_id": "111", "text_vector": gen_random_vector(10), "like": 1, "price": 1.11,
536533
# "author": ["gy"], "aim": True}
537534
# field2 = {"doc_id": "222", "text_vector": gen_random_vector(10), "like": 2, "price": 2.22,

‎volcengine/viking_db/Collection.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ def __init__(self, collection_name, fields, viking_db_service, primary_key, inde
3030
self.update_time = update_time
3131
if update_person is not None:
3232
self.update_person = update_person
33+
3334
def upsert_data(self, data: Union[Data, List[Data]], async_upsert=False):
3435
"""
3536
Insert and overwrite data in fields within a collection
@@ -47,7 +48,7 @@ def upsert_data(self, data: Union[Data, List[Data]], async_upsert=False):
4748
if async_upsert:
4849
params["async"]=True
4950
# print(params)
50-
res = self.viking_db_service.json_exception("UpsertData", {}, json.dumps(params))
51+
res = self.viking_db_service._retry_request("UpsertData", {}, json.dumps(params))
5152
elif isinstance(data, list):
5253
fields_arr = []
5354
ttl = 0
@@ -63,7 +64,7 @@ def upsert_data(self, data: Union[Data, List[Data]], async_upsert=False):
6364
params = {"collection_name": self.collection_name, "fields": record[item], "ttl": item}
6465
if async_upsert:
6566
params["async"]=True
66-
res = self.viking_db_service.json_exception("UpsertData", {}, json.dumps(params))
67+
res = self.viking_db_service._retry_request("UpsertData", {}, json.dumps(params))
6768

6869
async def async_upsert_data(self, data: Union[Data, List[Data]], async_upsert=False):
6970
if isinstance(data, Data):
@@ -132,7 +133,7 @@ def fetch_data(self, id: Union[str, List[str], int, List[int]]):
132133
if isinstance(id, str) or isinstance(id, int):
133134
params = {"collection_name": self.collection_name, "primary_keys": id}
134135
# print(params)
135-
res = self.viking_db_service.get_body_exception("FetchData", {}, json.dumps(params))
136+
res = self.viking_db_service._retry_request("FetchData", {}, json.dumps(params))
136137
# print(res)
137138
# res是一个列表,只有一个元素
138139
res = json.loads(res)
@@ -141,7 +142,7 @@ def fetch_data(self, id: Union[str, List[str], int, List[int]]):
141142
return data
142143
elif isinstance(id, List):
143144
params = {"collection_name": self.collection_name, "primary_keys": id}
144-
res = self.viking_db_service.get_body_exception("FetchData", {}, json.dumps(params))
145+
res = self.viking_db_service._retry_request("FetchData", {}, json.dumps(params))
145146
res = json.loads(res)
146147
# print(res["data"],self.primary_key)
147148
datas = []

‎volcengine/viking_db/Index.py‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import json
33
from typing import Union, List
44

5-
from volcengine.viking_db.common import Data, VectorOrder, ScalarOrder
5+
from volcengine.viking_db.common import MAX_RETRIES, Data, VectorOrder, ScalarOrder
66

77

88
class Index(object):
@@ -30,7 +30,7 @@ def __init__(self, collection_name, index_name, vector_index, scalar_index, stat
3030
# print(col["data"]["primary_key"])
3131
self.primary_key = col["data"]["primary_key"]
3232

33-
def search(self, order=None, filter=None, limit=10, output_fields=None, partition="default", dense_weight=None):
33+
def search(self, order=None, filter=None, limit=10, output_fields=None, partition="default", dense_weight=None, retry=False):
3434
"""
3535
Search for vectors or scalars similar to a given vector or scalar.
3636
@@ -51,10 +51,10 @@ def search(self, order=None, filter=None, limit=10, output_fields=None, partitio
5151
if order.vector is not None:
5252
res = self.search_by_vector(order.vector, sparse_vectors=order.sparse_vectors, filter=filter,
5353
limit=limit,
54-
output_fields=output_fields, partition=partition, dense_weight=dense_weight)
54+
output_fields=output_fields, partition=partition, dense_weight=dense_weight, retry=retry)
5555
elif order.id is not None:
5656
res = self.search_by_id(order.id, filter=filter, limit=limit,
57-
output_fields=output_fields, partition=partition, dense_weight=dense_weight)
57+
output_fields=output_fields, partition=partition, dense_weight=dense_weight, retry=retry)
5858
return res
5959
elif isinstance(order, ScalarOrder):
6060
search = {}
@@ -66,7 +66,8 @@ def search(self, order=None, filter=None, limit=10, output_fields=None, partitio
6666
search['filter'] = filter
6767
params = {"collection_name": self.collection_name, "index_name": self.index_name, "search": search}
6868
# print(params)
69-
res = self.viking_db_service.json_exception("SearchIndex", {}, json.dumps(params))
69+
remaining_retries = MAX_RETRIES if retry else 0
70+
res = self.viking_db_service._retry_request("SearchIndex", {}, json.dumps(params), remaining_retries)
7071
res = json.loads(res)
7172
# print(res["data"])
7273

@@ -91,7 +92,8 @@ def search(self, order=None, filter=None, limit=10, output_fields=None, partitio
9192
if filter is not None:
9293
search['filter'] = filter
9394
params = {"collection_name": self.collection_name, "index_name": self.index_name, "search": search}
94-
res = self.viking_db_service.json_exception("SearchIndex", {}, json.dumps(params))
95+
remaining_retries = MAX_RETRIES if retry else 0
96+
res = self.viking_db_service._retry_request("SearchIndex", {}, json.dumps(params), remaining_retries)
9597
res = json.loads(res)
9698

9799
datas = []
@@ -173,7 +175,7 @@ async def async_search(self, order=None, filter=None, limit=10, output_fields=No
173175
# print("==================")
174176
return datas
175177

176-
def search_by_id(self, id, filter=None, limit=10, output_fields=None, partition="default", dense_weight=None):
178+
def search_by_id(self, id, filter=None, limit=10, output_fields=None, partition="default", dense_weight=None, retry=False):
177179
"""
178180
Search for vectors similar to a given vector based on its id.
179181
@@ -200,7 +202,8 @@ def search_by_id(self, id, filter=None, limit=10, output_fields=None, partition=
200202
search['dense_weight'] = dense_weight
201203
params = {"collection_name": self.collection_name, "index_name": self.index_name, "search": search}
202204
# print(params)
203-
res = self.viking_db_service.json_exception("SearchIndex", {}, json.dumps(params))
205+
remaining_retries = MAX_RETRIES if retry else 0
206+
res = self.viking_db_service._retry_request("SearchIndex", {}, json.dumps(params), remaining_retries)
204207
res = json.loads(res)
205208
# print(res["data"])
206209

@@ -254,7 +257,7 @@ async def async_search_by_id(self, id, filter=None, limit=10, output_fields=None
254257
return datas
255258

256259
def search_by_vector(self, vector, sparse_vectors=None, filter=None, limit=10, output_fields=None,
257-
partition="default", dense_weight=None):
260+
partition="default", dense_weight=None, retry=False):
258261
"""
259262
Search for vectors similar to a given vector.
260263
@@ -284,7 +287,8 @@ def search_by_vector(self, vector, sparse_vectors=None, filter=None, limit=10, o
284287
search['dense_weight'] = dense_weight
285288
params = {"collection_name": self.collection_name, "index_name": self.index_name, "search": search}
286289
# print(params)
287-
res = self.viking_db_service.json_exception("SearchIndex", {}, json.dumps(params))
290+
remaining_retries = MAX_RETRIES if retry else 0
291+
res = self.viking_db_service._retry_request("SearchIndex", {}, json.dumps(params), remaining_retries)
288292
res = json.loads(res)
289293
# print(res["data"])
290294

@@ -338,7 +342,7 @@ async def async_search_by_vector(self, vector, sparse_vectors=None, filter=None,
338342
# print("==================")
339343
return datas
340344

341-
def search_by_text(self, text, filter=None, limit=10, output_fields=None, partition="default", dense_weight=None):
345+
def search_by_text(self, text, filter=None, limit=10, output_fields=None, partition="default", dense_weight=None, retry=False):
342346
"""
343347
Search for text similar to a given text.
344348
@@ -364,7 +368,8 @@ def search_by_text(self, text, filter=None, limit=10, output_fields=None, partit
364368
if dense_weight is not None:
365369
search['dense_weight'] = dense_weight
366370
params = {"collection_name": self.collection_name, "index_name": self.index_name, "search": search}
367-
res = self.viking_db_service.json_exception("SearchIndex", {}, json.dumps(params))
371+
remaining_retries = MAX_RETRIES if retry else 0
372+
res = self.viking_db_service._retry_request("SearchIndex", {}, json.dumps(params), remaining_retries)
368373
res = json.loads(res)
369374
# print(res["data"])
370375

@@ -429,7 +434,7 @@ def fetch_data(self, id: Union[str, List[str], int, List[int]], output_fields=No
429434
params["output_fields"] = output_fields
430435
if partition != "":
431436
params["partition"] = partition
432-
res = self.viking_db_service.get_body_exception("FetchIndexData", {}, json.dumps(params))
437+
res = self.viking_db_service._retry_request("FetchIndexData", {}, json.dumps(params))
433438
res = json.loads(res)
434439
# res["data"]是一个list
435440
# print(res["data"][0]["fields"])
@@ -446,7 +451,7 @@ def fetch_data(self, id: Union[str, List[str], int, List[int]], output_fields=No
446451
params["output_fields"] = output_fields
447452
if partition != "":
448453
params["partition"] = partition
449-
res = self.viking_db_service.get_body_exception("FetchIndexData", {}, json.dumps(params))
454+
res = self.viking_db_service._retry_request("FetchIndexData", {}, json.dumps(params))
450455
res = json.loads(res)
451456
# print(res)
452457
for item in res["data"]:

0 commit comments

Comments
 (0)