From 7b48f31bb7d303bb184321aea35d9c6fa6a50b93 Mon Sep 17 00:00:00 2001 From: Crimson <39024757+crimson-gao@users.noreply.github.com> Date: Fri, 26 Jul 2024 17:50:45 +0800 Subject: [PATCH 1/4] fix: pull resp time_as_str not apply on time nano (#300) --- aliyun/log/pulllog_response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aliyun/log/pulllog_response.py b/aliyun/log/pulllog_response.py index 5f74387..663a237 100755 --- a/aliyun/log/pulllog_response.py +++ b/aliyun/log/pulllog_response.py @@ -170,7 +170,7 @@ def loggroups_to_flattern_list(loggroup_list, time_as_str=None, decode_bytes=Non u'__topic__': logGroup.Topic, u'__source__': logGroup.Source} if log.Time_ns: - item[u'__time_ns_part__'] = log.Time_ns + item[u'__time_ns_part__'] = six.text_type(log.Time_ns) if time_as_str else log.Time_ns, item.update(tags) for content in log.Contents: item[PullLogResponse._b2u(content.Key) if decode_bytes else content.Key] = PullLogResponse._b2u(content.Value) if decode_bytes else content.Value From 7a38aa7aad30f9cd3825a3f52dbce2fb94324f3d Mon Sep 17 00:00:00 2001 From: Crimson <39024757+crimson-gao@users.noreply.github.com> Date: Mon, 29 Jul 2024 18:37:52 +0800 Subject: [PATCH 2/4] Fix compatiablity with python 2, add build test ci (#299) * support py2 pb * add ci test * update ci * ci * add utf8 encoding decl * fix py2 not compatible * merge import * fix: legacy module expt msg * add tag to test * add utf8 enc decl * use LogTags * fix ut * rename _proto_py2 * update packages --- .github/workflows/build.yaml | 40 +++ .github/workflows/py2-build.yaml | 35 +++ aliyun/log/__init__.py | 2 +- aliyun/log/_proto_py2/__init__.py | 9 + aliyun/log/_proto_py2/log_logs_pb2.py | 297 ++++++++++++++++++++++ aliyun/log/_proto_py2/log_logs_raw_pb2.py | 297 ++++++++++++++++++++++ aliyun/log/es_migration/migration_task.py | 2 +- aliyun/log/logclient.py | 3 +- aliyun/log/odps_sink.py | 2 +- aliyun/log/proto.py | 19 ++ aliyun/log/pulllog_response.py | 3 +- aliyun/log/version.py | 2 +- setup.py | 16 +- tests/ci/build-test/test_proto_use.py | 54 ++++ tests/ci/build-test/test_type.py | 8 + 15 files changed, 780 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/build.yaml create mode 100644 .github/workflows/py2-build.yaml create mode 100644 aliyun/log/_proto_py2/__init__.py create mode 100644 aliyun/log/_proto_py2/log_logs_pb2.py create mode 100644 aliyun/log/_proto_py2/log_logs_raw_pb2.py create mode 100644 aliyun/log/proto.py create mode 100644 tests/ci/build-test/test_proto_use.py create mode 100644 tests/ci/build-test/test_type.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..f4f77ab --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,40 @@ +name: Py3-build-test + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + test: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + python-version: [3.7, 3.12] + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -V + python -m pip install --upgrade pip setuptools + python -m pip install . + + - name: Show dependencies + run: | + python -m pip freeze + + - name: Run build tests + run: | + python -m pip install pytest + python -m pytest tests/ci/build-test/ \ No newline at end of file diff --git a/.github/workflows/py2-build.yaml b/.github/workflows/py2-build.yaml new file mode 100644 index 0000000..d9db57c --- /dev/null +++ b/.github/workflows/py2-build.yaml @@ -0,0 +1,35 @@ + + +name: Py2-build-test + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + test: + runs-on: ubuntu-20.04 + container: + image: python:2.7.18-buster + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + + - name: Install dependencies + run: | + python -V + python -m pip install --upgrade pip setuptools + python -m pip install . + + - name: Show dependencies + run: | + python -m pip freeze + + - name: Run build tests + run: | + python -m pip install pytest + python -m pytest tests/ci/build-test/ \ No newline at end of file diff --git a/aliyun/log/__init__.py b/aliyun/log/__init__.py index d6f863e..df7a5d6 100755 --- a/aliyun/log/__init__.py +++ b/aliyun/log/__init__.py @@ -33,7 +33,7 @@ from .shipper_response import * from .common_response import * from .external_store_config_response import * -from .log_logs_raw_pb2 import LogGroupRaw as LogGroup +from .proto import LogGroupRaw as LogGroup # logging handler from .logger_hanlder import SimpleLogHandler, QueuedLogHandler, LogFields, UwsgiQueuedLogHandler diff --git a/aliyun/log/_proto_py2/__init__.py b/aliyun/log/_proto_py2/__init__.py new file mode 100644 index 0000000..781cc90 --- /dev/null +++ b/aliyun/log/_proto_py2/__init__.py @@ -0,0 +1,9 @@ +import sys + +MAX_PYTHON_VERSION = (3, 0) + +if sys.version_info >= MAX_PYTHON_VERSION: + raise RuntimeError("This module works only on Python 2, " + "You are using Python {}.{}.{}".format(sys.version_info.major, + sys.version_info.minor, + sys.version_info.micro)) diff --git a/aliyun/log/_proto_py2/log_logs_pb2.py b/aliyun/log/_proto_py2/log_logs_pb2.py new file mode 100644 index 0000000..031ebe8 --- /dev/null +++ b/aliyun/log/_proto_py2/log_logs_pb2.py @@ -0,0 +1,297 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: log_logs.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='log_logs.proto', + package='', + syntax='proto2', + serialized_options=None, + create_key=_descriptor._internal_create_key, + serialized_pb=b'\n\x0elog_logs.proto\"k\n\x03Log\x12\x0c\n\x04Time\x18\x01 \x02(\r\x12\x1e\n\x08\x43ontents\x18\x02 \x03(\x0b\x32\x0c.Log.Content\x12\x0f\n\x07Time_ns\x18\x04 \x01(\x07\x1a%\n\x07\x43ontent\x12\x0b\n\x03Key\x18\x01 \x02(\t\x12\r\n\x05Value\x18\x02 \x02(\t\"$\n\x06LogTag\x12\x0b\n\x03Key\x18\x01 \x02(\t\x12\r\n\x05Value\x18\x02 \x02(\t\"~\n\x08LogGroup\x12\x12\n\x04Logs\x18\x01 \x03(\x0b\x32\x04.Log\x12\x10\n\x08Reserved\x18\x02 \x01(\t\x12\r\n\x05Topic\x18\x03 \x01(\t\x12\x0e\n\x06Source\x18\x04 \x01(\t\x12\x13\n\x0bMachineUUID\x18\x05 \x01(\t\x12\x18\n\x07LogTags\x18\x06 \x03(\x0b\x32\x07.LogTag\",\n\x0cLogGroupList\x12\x1c\n\tLogGroups\x18\x01 \x03(\x0b\x32\t.LogGroup' +) + + + + +_LOG_CONTENT = _descriptor.Descriptor( + name='Content', + full_name='Log.Content', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='Key', full_name='Log.Content.Key', index=0, + number=1, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='Value', full_name='Log.Content.Value', index=1, + number=2, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=88, + serialized_end=125, +) + +_LOG = _descriptor.Descriptor( + name='Log', + full_name='Log', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='Time', full_name='Log.Time', index=0, + number=1, type=13, cpp_type=3, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='Contents', full_name='Log.Contents', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='Time_ns', full_name='Log.Time_ns', index=2, + number=4, type=7, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_LOG_CONTENT, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=18, + serialized_end=125, +) + + +_LOGTAG = _descriptor.Descriptor( + name='LogTag', + full_name='LogTag', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='Key', full_name='LogTag.Key', index=0, + number=1, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='Value', full_name='LogTag.Value', index=1, + number=2, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=127, + serialized_end=163, +) + + +_LOGGROUP = _descriptor.Descriptor( + name='LogGroup', + full_name='LogGroup', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='Logs', full_name='LogGroup.Logs', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='Reserved', full_name='LogGroup.Reserved', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='Topic', full_name='LogGroup.Topic', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='Source', full_name='LogGroup.Source', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='MachineUUID', full_name='LogGroup.MachineUUID', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='LogTags', full_name='LogGroup.LogTags', index=5, + number=6, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=165, + serialized_end=291, +) + + +_LOGGROUPLIST = _descriptor.Descriptor( + name='LogGroupList', + full_name='LogGroupList', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='LogGroups', full_name='LogGroupList.LogGroups', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=293, + serialized_end=337, +) + +_LOG_CONTENT.containing_type = _LOG +_LOG.fields_by_name['Contents'].message_type = _LOG_CONTENT +_LOGGROUP.fields_by_name['Logs'].message_type = _LOG +_LOGGROUP.fields_by_name['LogTags'].message_type = _LOGTAG +_LOGGROUPLIST.fields_by_name['LogGroups'].message_type = _LOGGROUP +DESCRIPTOR.message_types_by_name['Log'] = _LOG +DESCRIPTOR.message_types_by_name['LogTag'] = _LOGTAG +DESCRIPTOR.message_types_by_name['LogGroup'] = _LOGGROUP +DESCRIPTOR.message_types_by_name['LogGroupList'] = _LOGGROUPLIST +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +Log = _reflection.GeneratedProtocolMessageType('Log', (_message.Message,), { + + 'Content' : _reflection.GeneratedProtocolMessageType('Content', (_message.Message,), { + 'DESCRIPTOR' : _LOG_CONTENT, + '__module__' : 'log_logs_pb2' + # @@protoc_insertion_point(class_scope:Log.Content) + }) + , + 'DESCRIPTOR' : _LOG, + '__module__' : 'log_logs_pb2' + # @@protoc_insertion_point(class_scope:Log) + }) +_sym_db.RegisterMessage(Log) +_sym_db.RegisterMessage(Log.Content) + +LogTag = _reflection.GeneratedProtocolMessageType('LogTag', (_message.Message,), { + 'DESCRIPTOR' : _LOGTAG, + '__module__' : 'log_logs_pb2' + # @@protoc_insertion_point(class_scope:LogTag) + }) +_sym_db.RegisterMessage(LogTag) + +LogGroup = _reflection.GeneratedProtocolMessageType('LogGroup', (_message.Message,), { + 'DESCRIPTOR' : _LOGGROUP, + '__module__' : 'log_logs_pb2' + # @@protoc_insertion_point(class_scope:LogGroup) + }) +_sym_db.RegisterMessage(LogGroup) + +LogGroupList = _reflection.GeneratedProtocolMessageType('LogGroupList', (_message.Message,), { + 'DESCRIPTOR' : _LOGGROUPLIST, + '__module__' : 'log_logs_pb2' + # @@protoc_insertion_point(class_scope:LogGroupList) + }) +_sym_db.RegisterMessage(LogGroupList) + + +# @@protoc_insertion_point(module_scope) diff --git a/aliyun/log/_proto_py2/log_logs_raw_pb2.py b/aliyun/log/_proto_py2/log_logs_raw_pb2.py new file mode 100644 index 0000000..4cc0b68 --- /dev/null +++ b/aliyun/log/_proto_py2/log_logs_raw_pb2.py @@ -0,0 +1,297 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: log_logs_raw.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='log_logs_raw.proto', + package='', + syntax='proto2', + serialized_options=None, + create_key=_descriptor._internal_create_key, + serialized_pb=b'\n\x12log_logs_raw.proto\"q\n\x06LogRaw\x12\x0c\n\x04Time\x18\x01 \x02(\r\x12!\n\x08\x43ontents\x18\x02 \x03(\x0b\x32\x0f.LogRaw.Content\x12\x0f\n\x07Time_ns\x18\x04 \x01(\x07\x1a%\n\x07\x43ontent\x12\x0b\n\x03Key\x18\x01 \x02(\t\x12\r\n\x05Value\x18\x02 \x02(\x0c\"\'\n\tLogTagRaw\x12\x0b\n\x03Key\x18\x01 \x02(\t\x12\r\n\x05Value\x18\x02 \x02(\t\"\x87\x01\n\x0bLogGroupRaw\x12\x15\n\x04Logs\x18\x01 \x03(\x0b\x32\x07.LogRaw\x12\x10\n\x08Reserved\x18\x02 \x01(\t\x12\r\n\x05Topic\x18\x03 \x01(\t\x12\x0e\n\x06Source\x18\x04 \x01(\t\x12\x13\n\x0bMachineUUID\x18\x05 \x01(\t\x12\x1b\n\x07LogTags\x18\x06 \x03(\x0b\x32\n.LogTagRaw\"2\n\x0fLogGroupListRaw\x12\x1f\n\tLogGroups\x18\x01 \x03(\x0b\x32\x0c.LogGroupRaw' +) + + + + +_LOGRAW_CONTENT = _descriptor.Descriptor( + name='Content', + full_name='LogRaw.Content', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='Key', full_name='LogRaw.Content.Key', index=0, + number=1, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='Value', full_name='LogRaw.Content.Value', index=1, + number=2, type=12, cpp_type=9, label=2, + has_default_value=False, default_value=b"", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=98, + serialized_end=135, +) + +_LOGRAW = _descriptor.Descriptor( + name='LogRaw', + full_name='LogRaw', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='Time', full_name='LogRaw.Time', index=0, + number=1, type=13, cpp_type=3, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='Contents', full_name='LogRaw.Contents', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='Time_ns', full_name='LogRaw.Time_ns', index=2, + number=4, type=7, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_LOGRAW_CONTENT, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=22, + serialized_end=135, +) + + +_LOGTAGRAW = _descriptor.Descriptor( + name='LogTagRaw', + full_name='LogTagRaw', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='Key', full_name='LogTagRaw.Key', index=0, + number=1, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='Value', full_name='LogTagRaw.Value', index=1, + number=2, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=137, + serialized_end=176, +) + + +_LOGGROUPRAW = _descriptor.Descriptor( + name='LogGroupRaw', + full_name='LogGroupRaw', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='Logs', full_name='LogGroupRaw.Logs', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='Reserved', full_name='LogGroupRaw.Reserved', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='Topic', full_name='LogGroupRaw.Topic', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='Source', full_name='LogGroupRaw.Source', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='MachineUUID', full_name='LogGroupRaw.MachineUUID', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='LogTags', full_name='LogGroupRaw.LogTags', index=5, + number=6, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=179, + serialized_end=314, +) + + +_LOGGROUPLISTRAW = _descriptor.Descriptor( + name='LogGroupListRaw', + full_name='LogGroupListRaw', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='LogGroups', full_name='LogGroupListRaw.LogGroups', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=316, + serialized_end=366, +) + +_LOGRAW_CONTENT.containing_type = _LOGRAW +_LOGRAW.fields_by_name['Contents'].message_type = _LOGRAW_CONTENT +_LOGGROUPRAW.fields_by_name['Logs'].message_type = _LOGRAW +_LOGGROUPRAW.fields_by_name['LogTags'].message_type = _LOGTAGRAW +_LOGGROUPLISTRAW.fields_by_name['LogGroups'].message_type = _LOGGROUPRAW +DESCRIPTOR.message_types_by_name['LogRaw'] = _LOGRAW +DESCRIPTOR.message_types_by_name['LogTagRaw'] = _LOGTAGRAW +DESCRIPTOR.message_types_by_name['LogGroupRaw'] = _LOGGROUPRAW +DESCRIPTOR.message_types_by_name['LogGroupListRaw'] = _LOGGROUPLISTRAW +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +LogRaw = _reflection.GeneratedProtocolMessageType('LogRaw', (_message.Message,), { + + 'Content' : _reflection.GeneratedProtocolMessageType('Content', (_message.Message,), { + 'DESCRIPTOR' : _LOGRAW_CONTENT, + '__module__' : 'log_logs_raw_pb2' + # @@protoc_insertion_point(class_scope:LogRaw.Content) + }) + , + 'DESCRIPTOR' : _LOGRAW, + '__module__' : 'log_logs_raw_pb2' + # @@protoc_insertion_point(class_scope:LogRaw) + }) +_sym_db.RegisterMessage(LogRaw) +_sym_db.RegisterMessage(LogRaw.Content) + +LogTagRaw = _reflection.GeneratedProtocolMessageType('LogTagRaw', (_message.Message,), { + 'DESCRIPTOR' : _LOGTAGRAW, + '__module__' : 'log_logs_raw_pb2' + # @@protoc_insertion_point(class_scope:LogTagRaw) + }) +_sym_db.RegisterMessage(LogTagRaw) + +LogGroupRaw = _reflection.GeneratedProtocolMessageType('LogGroupRaw', (_message.Message,), { + 'DESCRIPTOR' : _LOGGROUPRAW, + '__module__' : 'log_logs_raw_pb2' + # @@protoc_insertion_point(class_scope:LogGroupRaw) + }) +_sym_db.RegisterMessage(LogGroupRaw) + +LogGroupListRaw = _reflection.GeneratedProtocolMessageType('LogGroupListRaw', (_message.Message,), { + 'DESCRIPTOR' : _LOGGROUPLISTRAW, + '__module__' : 'log_logs_raw_pb2' + # @@protoc_insertion_point(class_scope:LogGroupListRaw) + }) +_sym_db.RegisterMessage(LogGroupListRaw) + + +# @@protoc_insertion_point(module_scope) diff --git a/aliyun/log/es_migration/migration_task.py b/aliyun/log/es_migration/migration_task.py index 8bd974f..2e9c692 100644 --- a/aliyun/log/es_migration/migration_task.py +++ b/aliyun/log/es_migration/migration_task.py @@ -192,7 +192,7 @@ def _run(self, shutdown_flag): scroll=self._es_scroll, ) except NotFoundError: - msg = f"cache is expired, which is with duration {self._es_scroll}" + msg = "cache is expired, which is with duration {}".format(self._es_scroll) self._logger.error(msg, exc_info=True) raise Exception(msg) diff --git a/aliyun/log/logclient.py b/aliyun/log/logclient.py index e9ae019..90949cc 100644 --- a/aliyun/log/logclient.py +++ b/aliyun/log/logclient.py @@ -1,3 +1,4 @@ +# -*- encoding: utf-8 -*- """ LogClient class is the main class in the SDK. It can be used to communicate with log service server to put/get data. @@ -50,7 +51,7 @@ from .util import base64_encodestring as e64, base64_decodestring as d64, Util from .version import API_VERSION, USER_AGENT -from .log_logs_raw_pb2 import LogGroupRaw as LogGroup +from .proto import LogGroupRaw as LogGroup from .external_store_config_response import * import struct from .logresponse import LogResponse diff --git a/aliyun/log/odps_sink.py b/aliyun/log/odps_sink.py index 957d4d9..3ac7d07 100644 --- a/aliyun/log/odps_sink.py +++ b/aliyun/log/odps_sink.py @@ -72,7 +72,7 @@ def setFields(self, fields): def getPartitionColumn(self): return self.__params["partitionColumn"] - def setPartitionColumn(self, partitionColumn: list): + def setPartitionColumn(self, partitionColumn): self.__params["partitionColumn"] = partitionColumn def getParams(self): diff --git a/aliyun/log/proto.py b/aliyun/log/proto.py new file mode 100644 index 0000000..75671d6 --- /dev/null +++ b/aliyun/log/proto.py @@ -0,0 +1,19 @@ +import sys + +if sys.version_info[0] == 2: + from ._proto_py2.log_logs_raw_pb2 import LogGroupListRaw, LogGroupRaw, LogTagRaw, LogRaw + from ._proto_py2.log_logs_pb2 import LogGroupList, LogGroup, LogTag, Log +else: + from .log_logs_raw_pb2 import LogGroupListRaw, LogGroupRaw, LogTagRaw, LogRaw + from .log_logs_pb2 import LogGroupList, LogGroup, LogTag, Log + +__all__ = [ + 'LogGroupListRaw', + 'LogGroupRaw', + 'LogTagRaw', + 'LogRaw', + 'LogGroupList', + 'LogGroup', + 'LogTag', + 'Log' +] \ No newline at end of file diff --git a/aliyun/log/pulllog_response.py b/aliyun/log/pulllog_response.py index 663a237..e90c56d 100755 --- a/aliyun/log/pulllog_response.py +++ b/aliyun/log/pulllog_response.py @@ -9,8 +9,7 @@ from .util import Util from .util import base64_encodestring as b64e -from .log_logs_pb2 import LogGroupList -from .log_logs_raw_pb2 import LogGroupListRaw +from .proto import LogGroupList, LogGroupListRaw import six DEFAULT_DECODE_LIST = ('utf8',) diff --git a/aliyun/log/version.py b/aliyun/log/version.py index a245fc4..6d11451 100644 --- a/aliyun/log/version.py +++ b/aliyun/log/version.py @@ -1,4 +1,4 @@ -__version__ = '0.9.7' +__version__ = '0.9.8' import sys OS_VERSION = str(sys.platform) diff --git a/setup.py b/setup.py index e0d3a27..1b7e54f 100755 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ 'elasticsearch', 'jmespath', 'dateparser', - 'protobuf>3.4.0,<4.0.0', + 'protobuf>3.20.3,<6.0.0', ] requirements_py2 = [ 'six==1.14.0', @@ -42,11 +42,19 @@ 'jmespath==0.9.5', 'enum34==1.1.10', 'futures==3.3.0', - 'protobuf>3.4.0,<4.0.0', + 'protobuf>3.4.0,<=3.17.3', # 3.18.0 is the last version support py2, but yanked 'regex==2021.3.17', 'tzlocal==2.0.0', 'lz4a==0.7.0', ] + +test_requirements = [ + 'pytest', + 'lz4', + 'virtualenv', + 'zstandard' +] + requirements = [] if sys.version_info[:2] == (2, 6): requirements = requirements_py2 @@ -67,6 +75,7 @@ 'aliyun.log.etl_core.trans_comp', 'aliyun.log.consumer', 'aliyun.log.es_migration', + 'aliyun.log._proto_py2', ] version = '' @@ -104,4 +113,7 @@ packages=packages, classifiers=classifiers, long_description=long_description, + extras_require = { + 'test': test_requirements, + }, ) diff --git a/tests/ci/build-test/test_proto_use.py b/tests/ci/build-test/test_proto_use.py new file mode 100644 index 0000000..1889f4d --- /dev/null +++ b/tests/ci/build-test/test_proto_use.py @@ -0,0 +1,54 @@ +# -*- encoding: utf-8 -*- +from aliyun.log.proto import * + + +def test_pb_encode(): + l = LogGroupList() + log_group = l.LogGroups.add() + log_group.Topic = "test" + log = log_group.Logs.add() + log.Time = 1721908188 + log.Time_ns = 1 + content = log.Contents.add() + content.Key = "a" + content.Value = "b" + tag = log_group.LogTags.add() + tag.Key = u"c中文" + tag.Value = u"d中文" + str = l.SerializeToString() + + d = LogGroupList() + d.ParseFromString(str) + assert d.LogGroups[0].Topic == "test" + assert d.LogGroups[0].Logs[0].Time == 1721908188 + assert d.LogGroups[0].Logs[0].Time_ns == 1 + assert d.LogGroups[0].Logs[0].Contents[0].Key == "a" + assert d.LogGroups[0].Logs[0].Contents[0].Value == "b" + assert d.LogGroups[0].LogTags[0].Key == u"c中文" + assert d.LogGroups[0].LogTags[0].Value == u"d中文" + + +def test_pb_raw_encode(): + l = LogGroupListRaw() + log_group = l.LogGroups.add() + log_group.Topic = "test" + log = log_group.Logs.add() + log.Time = 1721908188 + log.Time_ns = 1 + content = log.Contents.add() + content.Key = "a" + content.Value = b"b" + tag = log_group.LogTags.add() + tag.Key = u"c中文" + tag.Value = u"d中文" + str = l.SerializeToString() + + d = LogGroupListRaw() + d.ParseFromString(str) + assert d.LogGroups[0].Topic == "test" + assert d.LogGroups[0].Logs[0].Time == 1721908188 + assert d.LogGroups[0].Logs[0].Time_ns == 1 + assert d.LogGroups[0].Logs[0].Contents[0].Key == "a" + assert d.LogGroups[0].Logs[0].Contents[0].Value == b"b" + assert d.LogGroups[0].LogTags[0].Key == u"c中文" + assert d.LogGroups[0].LogTags[0].Value == u"d中文" diff --git a/tests/ci/build-test/test_type.py b/tests/ci/build-test/test_type.py new file mode 100644 index 0000000..9d2fe48 --- /dev/null +++ b/tests/ci/build-test/test_type.py @@ -0,0 +1,8 @@ +from aliyun.log import * +from aliyun.log.consumer import * +from aliyun.log.etl_core import * +from aliyun.log.es_migration import * +from aliyun.log.ext import * + +def test_dummy(): + pass \ No newline at end of file From 97bd68f435646151411c36e9e6e5eb1f1c0e746f Mon Sep 17 00:00:00 2001 From: Crimson <39024757+crimson-gao@users.noreply.github.com> Date: Fri, 2 Aug 2024 15:32:09 +0800 Subject: [PATCH 3/4] support proto 3.20.3 (#301) --- aliyun/log/version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aliyun/log/version.py b/aliyun/log/version.py index 6d11451..79c796e 100644 --- a/aliyun/log/version.py +++ b/aliyun/log/version.py @@ -1,4 +1,4 @@ -__version__ = '0.9.8' +__version__ = '0.9.9' import sys OS_VERSION = str(sys.platform) diff --git a/setup.py b/setup.py index 1b7e54f..2927c0a 100755 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ 'elasticsearch', 'jmespath', 'dateparser', - 'protobuf>3.20.3,<6.0.0', + 'protobuf>=3.20.3,<6.0.0', ] requirements_py2 = [ 'six==1.14.0', From 9552eef2a74b18e6e4463f68878ae063ac5b2485 Mon Sep 17 00:00:00 2001 From: zeyu-zhang <61429689+zeyu-zh@users.noreply.github.com> Date: Fri, 9 Aug 2024 16:45:39 +0800 Subject: [PATCH 4/4] fix: parse tuple credentials (#302) * Enhance security by introducing credential providers Implemented the concepts of 'credentials' and 'credential providers' to eliminate the need for hardcoding access keys in the codebase. Us- ers can now securely obtain credentials via third-party SDKs, suppo- rting retrieval from services like KMS and environment variables. This enhancement aligns with best practices for cloud service interaction and adds flexibility for custom use cases. Signed-off-by: Zeyu Zhang * Retrieve credentials from providers Signed-off-by: Zeyu Zhang * fix: parse tuple credentials Signed-off-by: Zeyu Zhang * bump version to 0.9.10 Signed-off-by: Zeyu Zhang --------- Signed-off-by: Zeyu Zhang --- aliyun/log/logclient.py | 3 ++- aliyun/log/version.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/aliyun/log/logclient.py b/aliyun/log/logclient.py index 90949cc..90818fe 100644 --- a/aliyun/log/logclient.py +++ b/aliyun/log/logclient.py @@ -178,7 +178,8 @@ def _replace_credentials(self): self._last_refresh = time.time() for tries in range(DEFAULT_REFRESH_RETRY_COUNT + 1): try: - self._auth = make_auth(StaticCredentialsProvider(self._credentials_auto_refresher()), + credentials = self._credentials_auto_refresher() + self._auth = make_auth(StaticCredentialsProvider(credentials[0], credentials[1], credentials[2]), self._auth_version, self._region) except Exception as ex: logger.error( diff --git a/aliyun/log/version.py b/aliyun/log/version.py index 79c796e..c15a01f 100644 --- a/aliyun/log/version.py +++ b/aliyun/log/version.py @@ -1,4 +1,4 @@ -__version__ = '0.9.9' +__version__ = '0.9.10' import sys OS_VERSION = str(sys.platform)