Init commit

.gitignore

+.idea
\ No newline at end of file

Gemfile

+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in aliyun-log-ruby-sdk.gemspec
+gemspec

LICENSE.txt

+Copyright (c) 2018 Weiwenjia
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

+# 阿里云日志SDK
+
+### https://help.aliyun.com/product/28958.html?spm=a2c4g.750001.9.1.1f087b13e7djrJ
+
+### 依赖
+
+* Httparty ~> 0.13
+* beefcake ~> 1.2.0
+
+### 用例
+```ruby
+# config/initializers/aliyun_log.rb
+ 
+AliyunLogRubySdk.configure do |config|
+  config.endpoint = 'your endpoint'
+  config.access_key_id = 'your access_key_id'
+  config.access_key_secret = 'your access_key_secret'
+  config.project_name = 'your project_name'
+  config.log_store_name = 'your log_store_name'
+  config.debug = true # false 则不打印请求日志，默认值：false
+end
+ 
+# 初始化 client
+client = AliyunLogRubySdk::Client.new
+
+# 上传接口
+log_items = []
+
+1.upto(5) do |i|
+  log_item = AliyunLogRubySdk::LogItem.new
+  log_item.contents = [['content', "我是日志#{i}"]]
+
+  log_items << log_item
+end
+
+client.put_logs(topic: 'your topic', source: 'your source', log_items: log_items)
+
+# 拉取日志接口
+client.get_cursor(shard_id: your_shard_id, query: { from: a_timestamp })
+client.pull_logs(shard_id: your_shard_id, query: { cursor: '上个接口返回的 cursor ', count: count })
+```
\ No newline at end of file

aliyun_log_ruby_sdk.gemspec

+# -*- encoding : utf-8 -*-
+
+lib = File.expand_path('../lib', __FILE__)
+
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'aliyun_log_ruby_sdk/version'
+
+Gem::Specification.new do |spec|
+  # Basic information
+  spec.name        = 'aliyun_log_ruby_sdk'
+  spec.version     = AliyunLogRubySdk::VERSION
+  spec.date        = '2018-10-31'
+  spec.summary     = 'Aliyun log ruby sdk'
+  spec.description = 'Aliyun log ruby sdk (unofficial).'
+  spec.authors     = ['JXC']
+  spec.license     = 'MIT'
+  spec.homepage    = 'https://www.weiwenjia.com/'
+
+  # Require files
+  spec.files         = `git ls-files`.split($/)
+  spec.require_paths = ['lib']
+
+  # Development dependency
+  spec.add_development_dependency 'bundler', '~> 1.7'
+  spec.add_development_dependency 'rake', '~> 10.0'
+
+  # Runtime dependency
+  spec.add_runtime_dependency 'httparty', '~> 0.13'
+  spec.add_runtime_dependency 'beefcake', '~> 1.2.0'
+end

lib/aliyun_log_ruby_sdk.rb

+# -*- encoding : utf-8 -*-
+
+require 'aliyun_log_ruby_sdk/configuration'
+require 'aliyun_log_ruby_sdk/client'
+require 'aliyun_log_ruby_sdk/log_item'
+
+module AliyunLogRubySdk
+  class ConfigurationError < StandardError; end
+  class ParseDataError < StandardError; end
+  class PostBodyTooLarge < StandardError; end
+end
\ No newline at end of file

lib/aliyun_log_ruby_sdk/client.rb

+# -*- encoding : utf-8 -*-
+
+require 'httparty'
+
+require_relative 'headers'
+require_relative 'project'
+require_relative 'config'
+require_relative 'log_store'
+require_relative 'index'
+require_relative 'shard'
+require_relative 'shipper'
+require_relative 'consumer_group'
+require_relative 'protobuf'
+
+module AliyunLogRubySdk
+  class Client
+
+    include HTTParty
+
+    include Headers
+    include Project
+    include Config
+    include LogStore
+    include Index
+    include Shard
+    include Shipper
+    include ConsumerGroup
+
+    include Protobuf
+
+    delegate :endpoint,
+             :access_key_id,
+             :access_key_secret,
+             :project_name,
+             :log_store_name,
+             :debug,
+             to: :'AliyunLogRubySdk.config'
+
+    def initialize
+      self.class.base_uri(base_uri)
+    end
+
+    def get(path, options = {})
+      options[:headers] ||= {}
+
+      headers = generate_headers(headers: options[:headers],
+                                 resource: generate_resource(path, options[:query]))
+
+      response = self.class.get(path, query: options[:query], headers: headers)
+
+      if debug?
+        response_string = "GET #{base_uri}#{path}, query: #{options[:query].to_param}, headers: #{headers}"
+
+        response_string << ", response: #{response.parsed_response}" if options[:headers]['Content-Type'] != 'application/x-protobuf'
+
+        puts response_string
+      end
+
+      response
+    end
+
+    def post(path, options = {})
+      options[:headers] ||= {}
+      options[:body] ||= {}
+
+      if options[:headers]['Content-Type'] == 'application/x-protobuf'
+        body = options[:body]
+      else
+        body = options[:body].transform_keys do |key|
+          key.to_s.camelize(:lower)
+        end
+
+        body = body.to_json if body.present?
+      end
+
+      headers = generate_headers(verb: 'POST',
+                                 headers: options[:headers],
+                                 resource: generate_resource(path),
+                                 body: body)
+
+      response = self.class.post(path, headers: headers, body: body)
+
+      if debug?
+        response_string = "POST #{base_uri}#{path}, headers: #{headers}"
+
+        if options[:headers]['Content-Type'] != 'application/x-protobuf'
+          response_string << ", body: #{body}, response: #{response.parsed_response}"
+        end
+
+        puts response_string
+      end
+
+      response
+    end
+
+    def put(path, options = {})
+      raise NotImplementedError
+    end
+
+    def delete(path, options = {})
+      options[:body] ||= {}
+
+      body = options[:body].transform_keys do |key|
+        key.to_s.camelize(:lower)
+      end
+
+      body = body.to_json if body.present?
+
+      headers = generate_headers(verb: 'DELETE',
+                                 headers: options[:headers],
+                                 resource: generate_resource(path),
+                                 body: body)
+
+      response = self.class.delete(path, headers: headers, body: body)
+
+      puts "DELETE #{base_uri}#{path}, headers: #{headers}, body: #{body}, response: #{response.parsed_response}" if debug?
+
+      response
+    end
+
+    private
+
+    def base_uri
+      "http://#{project_name}.#{endpoint}"
+    end
+
+    def generate_resource(path, query = {})
+      # 这里有一个很奇葩的地方：如果参数的值中存在 =，to_param 方法会将其转变成 %3D，但是阿里云日志签名时是用 = 做签名，因此这里替换回来
+      query.present? ? "#{path}?#{query.to_param.gsub('%3D', '=')}" : path
+    end
+
+    def debug?
+      debug
+    end
+
+  end
+end
\ No newline at end of file

lib/aliyun_log_ruby_sdk/config.rb

+# -*- encoding : utf-8 -*-
+
+module AliyunLogRubySdk
+  module Config
+    def list_config(options = {})
+      get('/configs', options)
+    end
+
+    def create_config(options = {})
+      raise NotImplementedError
+    end
+
+    def delete_config(options = {})
+      raise NotImplementedError
+    end
+
+    def get_config(options = {})
+      get("/configs/#{options[:config_name]}", options)
+    end
+
+    def get_applied_machine_groups(options = {})
+      get("/configs/#{options[:config_name]}/machinegroups", options)
+    end
+  end
+end
\ No newline at end of file

lib/aliyun_log_ruby_sdk/configuration.rb

+# -*- encoding : utf-8 -*-
+
+module AliyunLogRubySdk
+  def self.configure
+    raise AliyunLogRubySdk::ConfigurationError, 'Block must be given' unless block_given?
+
+    yield @config ||= Configuration.new
+  end
+
+  def self.config
+    @config.debug = false if @config.debug.nil?
+
+    @config
+  end
+
+  class Configuration
+
+    attr_accessor :endpoint, :access_key_id, :access_key_secret, :project_name, :log_store_name, :debug
+
+  end
+end
\ No newline at end of file

lib/aliyun_log_ruby_sdk/consumer_group.rb

+# -*- encoding : utf-8 -*-
+
+module AliyunLogRubySdk
+  module ConsumerGroup
+    def list_consumer_group(options = {})
+      get("/logstores/#{options[:log_store_name]}/consumergroups", options)
+    end
+
+    def get_check_point(options = {})
+      get("/logstores/#{options[:log_store_name]}/consumergroups/#{options[:consumer_group]}", options)
+    end
+  end
+end
\ No newline at end of file

lib/aliyun_log_ruby_sdk/headers.rb

+# -*- encoding : utf-8 -*-
+
+require 'base64'
+require 'openssl'
+
+require_relative 'version'
+require_relative 'protobuf'
+
+module AliyunLogRubySdk
+  module Headers
+    X_LOG_SIGNATURE_METHOD = 'hmac-sha1'.freeze
+
+    def generate_headers(options = {})
+      @options = options
+
+      @headers = options[:headers] || {}
+      host = @headers.delete('x-log-host')
+
+      @headers = @headers.merge('x-log-apiversion' => X_LOG_API_VERSION,
+                                'x-log-signaturemethod' => X_LOG_SIGNATURE_METHOD,
+                                'Date' => Time.now.httpdate,
+                                'Host' => (host || "#{project_name}.#{endpoint}"))
+
+      body = options[:body]
+
+      if body.present?
+	      @headers['Content-Length'] = body.bytesize.to_s
+	      @headers['Content-MD5'] = calculate_md5(body)
+      else
+	      @headers['Content-Length'] = '0'
+	      @headers['x-log-bodyrawsize'] = '0'
+      end
+
+      @headers['Authorization'] = authorization
+
+      @headers['x-log-date'] = @headers['Date']
+
+      @headers
+    end
+
+    private
+
+    def authorization
+      "LOG #{access_key_id}:#{signature}"
+    end
+
+    def signature
+      Base64.encode64(OpenSSL::HMAC.digest('sha1', access_key_secret, sign_string.encode('utf-8'))).rstrip
+    end
+
+    def sign_string
+      %W[
+        #{@options[:verb] || 'GET'}
+        #{@headers['Content-MD5']}
+        #{@headers['Content-Type']}
+        #{@headers['Date']}
+        #{canonicalized_log_headers}
+        #{@options[:resource]}
+      ].join("\n")
+    end
+
+    def calculate_md5(body)
+      Digest::MD5.hexdigest(body).upcase
+    end
+
+    def canonicalized_log_headers
+      @headers.select { |key, _value| key.match(/x-log-*|x-acs-*/) }.map do |key, value|
+        "#{key.downcase}:#{value}"
+      end.sort.join("\n")
+    end
+  end
+end
\ No newline at end of file

lib/aliyun_log_ruby_sdk/index.rb

+# -*- encoding : utf-8 -*-
+
+module AliyunLogRubySdk
+  module Index
+    def get_index(options = {})
+      get("/logstores/#{options[:log_store_name]}/index", options)
+    end
+  end
+end
\ No newline at end of file

lib/aliyun_log_ruby_sdk/log_item.rb

+# -*- encoding : utf-8 -*-
+
+module AliyunLogRubySdk
+  class LogItem
+
+    attr_accessor :timestamp, :contents
+
+    def initialize(timestamp = Time.now.to_i, contents = [])
+      @timestamp = timestamp
+      @contents = contents
+    end
+
+  end
+end
\ No newline at end of file

lib/aliyun_log_ruby_sdk/log_store.rb

+# -*- encoding : utf-8 -*-
+
+module AliyunLogRubySdk
+  module LogStore
+    def list_log_store(options = {})
+      get('/logstores', options)
+    end
+
+    def create_log_store(options = {})
+      options[:headers] = { 'x-log-bodyrawsize' => '0', 'Content-Type' => 'application/json' }
+
+      post('/logstores', options)
+    end
+
+    def delete_log_store(options = {})
+      raise NotImplementedError
+
+	    # delete("/logstores/#{log_store_name}", options)
+    end
+
+    def get_log_store(options = {})
+      get("/logstores/#{log_store_name}", options)
+    end
+
+    def update_log_store(options = {})
+      raise NotImplementedError
+    end
+
+    def get_logs(options = {})
+      options[:query] ||= {}
+      options[:type] = 'log'
+
+      get("/logstores/#{log_store_name}", options)
+    end
+
+    def get_histograms(options = {})
+      options[:query] ||= {}
+      options[:type] = 'histogram'
+
+      get("/logstores/#{log_store_name}", options)
+    end
+  end
+end
\ No newline at end of file

lib/aliyun_log_ruby_sdk/machine_group.rb

+# -*- encoding : utf-8 -*-
+
+module AliyunLogRubySdk
+  module MachineGroup
+    def list_machine_group(options = {})
+      get('/machinegroups', options)
+    end
+
+    def create_machine_group(options = {})
+      raise NotImplementedError
+    end
+
+    def delete_machine_group(options = {})
+      raise NotImplementedError
+    end
+
+    def get_machine_group(options = {})
+      get("/machinegroups/#{options[:machine_group_name]}", options)
+    end
+  end
+end
\ No newline at end of file

lib/aliyun_log_ruby_sdk/project.rb

+# -*- encoding : utf-8 -*-
+
+module AliyunLogRubySdk
+  module Project
+    def list_project(options = {})
+      options[:headers] ||= {}
+      options[:headers]['x-log-host'] = endpoint
+
+      get('/', options)
+    end
+
+    def create_project(options = {})
+      raise NotImplementedError
+    end
+
+    def delete_project(options = {})
+      raise NotImplementedError
+    end
+
+    def get_project(options = {})
+      get('/', options)
+    end
+  end
+end
\ No newline at end of file

lib/aliyun_log_ruby_sdk/protobuf.rb

+# -*- encoding : utf-8 -*-
+
+require 'beefcake'
+
+# message Log
+# {
+#     required uint32 Time = 1;// UNIX Time Format
+#     message Content
+#     {
+#         required string Key = 1;
+#         required string Value = 2;
+#     }
+#     repeated Content Contents = 2;
+# }
+#
+# message LogTag
+# {
+#     required string Key = 1;
+#     required string Value = 2;
+# }
+#
+# message LogGroup
+# {
+#     repeated Log Logs = 1;
+#     optional string Reserved = 2; // reserved fields
+#     optional string Topic = 3;
+#     optional string Source = 4;
+#     repeated LogTag LogTags = 6;
+# }
+#
+# message LogGroupList
+# {
+#     repeated LogGroup logGroupList = 1;
+# }
+
+module AliyunLogRubySdk
+  module Protobuf
+    class Log
+      include Beefcake::Message
+
+      required :time, :uint32, 1
+
+      class Content
+        include Beefcake::Message
+
+        required :key, :string, 1
+        required :value, :string, 2
+      end
+
+      repeated :contents, Content, 2
+    end
+
+    class LogTag
+      include Beefcake::Message
+
+      required :key, :string, 1
+      required :value, :string, 2
+    end
+
+    class LogGroup
+      include Beefcake::Message
+
+      repeated :logs, Log, 1
+
+      optional :reserved, :string, 2
+      optional :topic, :string, 3
+      optional :source, :string, 4
+
+      repeated :log_tags, LogTag, 6
+    end
+
+    class LogGroupList
+      include Beefcake::Message
+
+      repeated :log_group_list, LogGroup, 1
+    end
+  end
+end
\ No newline at end of file

lib/aliyun_log_ruby_sdk/shard.rb

+# -*- encoding : utf-8 -*-
+
+module AliyunLogRubySdk
+  module Shard
+    def list_shards(options = {})
+      get("/logstores/#{log_store_name}/shards", options)
+    end
+
+    def get_cursor(options = {})
+      options[:query] ||= {}
+      options[:query][:type] = 'cursor'
+
+      get("/logstores/#{log_store_name}/shards/#{options[:shard_id]}", options)
+    end
+
+    def put_logs(options = {})
+      body = generate_log_group(options).encode.to_s
+
+      raise AliyunLogRubySdk::PostBodyTooLarge, 'Post body is larger then 3MB' if body.length > 3 * 1024 * 1024
+
+      options[:headers] = {
+        'Content-Type' => 'application/x-protobuf',
+	      'x-log-bodyrawsize' => body.length.to_s,
+	      'x-log-compresstype' => 'deflate'
+      }
+
+      options[:body] = Zlib::Deflate.deflate(body)
+
+      post("/logstores/#{log_store_name}/shards/lb", options)
+    end
+
+    def pull_logs(options = {})
+      options[:headers] = {
+        'Accept' => 'application/x-protobuf',
+        'Accept-Encoding' => 'gzip',
+        'Content-Type' => 'application/x-protobuf',
+      }
+
+      options[:query] ||= {}
+      options[:query][:type] = 'log'
+
+      parse(get("/logstores/#{log_store_name}/shards/#{options[:shard_id]}", options))
+    end
+
+    private
+
+    def generate_log_group(options)
+      log_group = Protobuf::LogGroup.new(logs: [], topic: options[:topic], source: options[:source])
+
+      options[:log_items].each do |log_item|
+        log = Protobuf::Log.new(time: log_item.timestamp, contents: [])
+
+        log_item.contents.each do |content|
+          log.contents << Protobuf::Log::Content.new(key: content[0], value: content[1])
+        end
+
+        log_group.logs << log
+      end
+
+      log_group
+    end
+
+    def parse(response)
+      begin
+        log_group_list = Protobuf::LogGroupList.decode(Zlib::Inflate.inflate(response.body))
+      rescue
+        raise AliyunLogRubySdk::ParseDataError, 'Failed to parse data to LogGroupList'
+      end
+
+      group_list = { detail: [] }
+
+      log_group_list.log_group_list.each do |log_group|
+        group = { topic: log_group.topic, source: log_group.source, logs: [] }
+
+        log_group.logs.each do |log|
+          l = { time: log.time, contents: [] }
+
+          log.contents.each do |content|
+            l[:contents] << [content.key, content.value]
+          end
+
+          group[:logs] << l
+        end
+
+        group_list[:detail] << group
+      end
+
+      puts "response: #{group_list.inspect}" if debug?
+    end
+  end
+end
\ No newline at end of file

lib/aliyun_log_ruby_sdk/shipper.rb

+# -*- encoding : utf-8 -*-
+
+module AliyunLogRubySdk
+  module Shipper
+    def get_shipper_status(options = {})
+      get("/logstores/#{options[:log_store_name]}/shipper/#{options[:shipper_name]}/tasks", options)
+    end
+  end
+end
\ No newline at end of file

lib/aliyun_log_ruby_sdk/version.rb

+# -*- encoding : utf-8 -*-
+
+module AliyunLogRubySdk
+  VERSION = '0.0.1'
+  X_LOG_API_VERSION = '0.6.0'
+end