diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 237d746ef..a6804f695 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,9 +25,9 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true +# - uses: satackey/action-docker-layer-caching@v0.0.11 +# # Ignore the failure of a step and avoid terminating the job. +# continue-on-error: true - name: Build and push id: docker_build uses: mr-smithers-excellent/docker-build-push@v5 diff --git a/app/controllers/api/v2/rgbpp_assets_statistics_controller.rb b/app/controllers/api/v2/rgbpp_assets_statistics_controller.rb new file mode 100644 index 000000000..ac8303481 --- /dev/null +++ b/app/controllers/api/v2/rgbpp_assets_statistics_controller.rb @@ -0,0 +1,24 @@ +module Api + module V2 + class RgbppAssetsStatisticsController < BaseController + def index + expires_in 15.minutes, public: true, stale_while_revalidate: 5.minutes, stale_if_error: 5.minutes + + statistics = RgbppAssetsStatistic.all.order(created_at_unixtimestamp: :asc) + statistics = statistics.where(network: params[:network]) if params[:network].present? + statistics = statistics.where(indicator: params[:indicators].split(",")) if params[:indicators].present? + + render json: { + data: statistics.map do |statistic| + { + indicator: statistic.indicator, + value: statistic.value.to_s, + network: statistic.network, + created_at_unixtimestamp: statistic.created_at_unixtimestamp.to_s, + } + end, + } + end + end + end +end diff --git a/app/controllers/api/v2/scripts_controller.rb b/app/controllers/api/v2/scripts_controller.rb index 3332a8fe0..9848fa024 100644 --- a/app/controllers/api/v2/scripts_controller.rb +++ b/app/controllers/api/v2/scripts_controller.rb @@ -21,7 +21,7 @@ def ckb_transactions base_query = CkbTransaction.joins(:cell_dependencies). where(cell_dependencies: { contract_cell_id: contract_cell_ids }). order("cell_dependencies.block_number DESC, cell_dependencies.tx_index DESC"). - limit(10000) + limit(Settings.query_default_limit) @ckb_transactions = CkbTransaction.from("(#{base_query.to_sql}) AS ckb_transactions"). order("block_number DESC, tx_index DESC"). page(@page). @@ -43,7 +43,7 @@ def referring_cells scope = Contract.referring_cells_query(@contracts). order("block_timestamp DESC, cell_index DESC"). - limit(10000) + limit(Settings.query_default_limit) if params[:args].present? type_script = TypeScript.find_by(args: params[:args]) scope = scope.or(CellOutput.where(type_script_id: type_script.id)) diff --git a/app/controllers/api/v2/udt_hourly_statistics_controller.rb b/app/controllers/api/v2/udt_hourly_statistics_controller.rb new file mode 100644 index 000000000..00ac149aa --- /dev/null +++ b/app/controllers/api/v2/udt_hourly_statistics_controller.rb @@ -0,0 +1,28 @@ +module Api + module V2 + class UdtHourlyStatisticsController < BaseController + def show + expires_in 15.minutes, public: true, stale_while_revalidate: 5.minutes, stale_if_error: 5.minutes + + udt = Udt.find_by!(type_hash: params[:id], published: true) + hourly_statistics = + if udt.present? + UdtHourlyStatistic.where(udt:).order(created_at_unixtimestamp: :asc) + else + UdtHourlyStatistic.none + end + + render json: { + data: hourly_statistics.map do |statistic| + { + ckb_transactions_count: statistic.ckb_transactions_count.to_s, + amount: statistic.amount.to_s, + holders_count: statistic.holders_count.to_s, + created_at_unixtimestamp: statistic.created_at_unixtimestamp.to_s, + } + end, + } + end + end + end +end diff --git a/app/interactions/addresses/ckb_transactions.rb b/app/interactions/addresses/ckb_transactions.rb index cb38536c6..c47373737 100644 --- a/app/interactions/addresses/ckb_transactions.rb +++ b/app/interactions/addresses/ckb_transactions.rb @@ -13,7 +13,7 @@ def execute raise AddressNotFoundError if address.is_a?(NullAddress) address_id = address.map(&:id) - account_books = AccountBook.where(address_id:).order("ckb_transaction_id desc").select(:ckb_transaction_id).distinct.limit(5000) + account_books = AccountBook.where(address_id:).order("ckb_transaction_id desc").select(:ckb_transaction_id).distinct.limit(Settings.query_default_limit) records = CkbTransaction.where(tx_status: :committed, id: account_books.map(&:ckb_transaction_id)).order(transactions_ordering).page(page).per(page_size) options = paginate_options(records, address_id) options.merge!(params: { previews: true, address: }) @@ -35,7 +35,7 @@ def transactions_ordering def paginate_options(records, address_id) total_count = AccountBook.where(address_id:).distinct.count FastJsonapi::PaginationMetaGenerator.new( - request:, records:, page:, page_size:, total_count:, + request:, records:, page:, page_size:, total_pages: records.total_pages, total_count:, ).call end diff --git a/app/lib/fast_jsonapi/pagination_meta_generator.rb b/app/lib/fast_jsonapi/pagination_meta_generator.rb index 09aad67f8..0104f61e7 100644 --- a/app/lib/fast_jsonapi/pagination_meta_generator.rb +++ b/app/lib/fast_jsonapi/pagination_meta_generator.rb @@ -3,19 +3,19 @@ class PaginationMetaGenerator DEFAULT_PAGE = 1 DEFAULT_PER_PAGE = 20 - def initialize(request:, records:, page:, page_size:, records_counter: nil, total_count: nil) + def initialize(request:, records:, page:, page_size:, total_pages: nil, records_counter: nil, total_count: nil) @url = request.base_url + request.path + query_string(request.query_parameters) @page = page.to_i @page_size = limit_page_size(records, page_size.to_i) @records = records @records_counter = records_counter || records @total_count = total_count || @records_counter.total_count.to_i - @total_pages = total_pages - @hash = { links: {}, meta: { total: @total_count, page_size: @page_size } } + @total_pages = total_pages || calculated_total_pages + @hash = { links: {}, meta: { total: @total_count, page_size: @page_size, total_pages: @total_pages } } end - def total_pages - (total_count / @page_size).ceil + def calculated_total_pages + (total_count.to_f / @page_size).ceil end def call @@ -31,10 +31,6 @@ def current_page records.current_page end - def last_page? - current_page == total_pages - end - def next_page current_page + 1 unless last_page? || out_of_range? end diff --git a/app/models/cell_dependency.rb b/app/models/cell_dependency.rb index 0729d5086..845adc6d1 100644 --- a/app/models/cell_dependency.rb +++ b/app/models/cell_dependency.rb @@ -26,12 +26,12 @@ def to_raw # Table name: cell_dependencies # # id :bigint not null, primary key +# contract_id :bigint # ckb_transaction_id :bigint not null # dep_type :integer # contract_cell_id :bigint not null # script_id :bigint -# contract_id :bigint -# implicit :boolean +# implicit :boolean default(TRUE), not null # block_number :bigint # tx_index :integer # contract_analyzed :boolean default(FALSE) @@ -40,6 +40,8 @@ def to_raw # # index_cell_dependencies_on_block_number_and_tx_index (block_number,tx_index) # index_cell_dependencies_on_contract_analyzed (contract_analyzed) +# index_cell_dependencies_on_contract_id (contract_id) +# index_cell_dependencies_on_script_id (script_id) # index_cell_dependencies_on_tx_id_and_cell_id_and_dep_type (ckb_transaction_id,contract_cell_id,dep_type) UNIQUE # index_on_cell_dependencies_contract_cell_block_tx (contract_cell_id,block_number DESC,tx_index DESC) # diff --git a/app/models/daily_statistic.rb b/app/models/daily_statistic.rb index ff2d11f17..40f3b863b 100644 --- a/app/models/daily_statistic.rb +++ b/app/models/daily_statistic.rb @@ -5,7 +5,7 @@ class DailyStatistic < ApplicationRecord transactions_count addresses_count total_dao_deposit live_cells_count dead_cells_count avg_hash_rate avg_difficulty uncle_rate total_depositors_count address_balance_distribution total_tx_fee occupied_capacity daily_dao_deposit daily_dao_depositors_count circulation_ratio daily_dao_withdraw nodes_count circulating_supply burnt locked_capacity treasury_amount mining_reward - deposit_compensation liquidity created_at_unixtimestamp ckb_hodl_wave holder_count knowledge_size + deposit_compensation liquidity created_at_unixtimestamp ckb_hodl_wave holder_count knowledge_size activity_address_contract_distribution ).freeze MILLISECONDS_IN_DAY = BigDecimal(24 * 60 * 60 * 1000) GENESIS_TIMESTAMP = 1573852190812 @@ -409,8 +409,26 @@ def liquidity dead_query = CellOutput.dead.generated_before(to_be_counted_date.to_i * 1000 - 1).consumed_after(to_be_counted_date.to_i * 1000).select(:address_id).to_sql combined_query = "#{live_query} UNION #{dead_query}" count_query = "SELECT COUNT(DISTINCT address_id) AS count FROM (#{combined_query}) AS combined_results;" - count = ActiveRecord::Base.connection.execute(count_query).first["count"] - count + ActiveRecord::Base.connection.execute(count_query).first["count"] + end + + define_logic :activity_address_contract_distribution do + block_ids = blocks_in_current_period.pluck(:id) + uniq_address_ids = CellOutput.established_status.where(block_id: block_ids).select(:address_id).distinct.map { |cell_output| cell_output.address_id } + results = Address.joins(:lock_script).where(id: uniq_address_ids).group(:code_hash).count + parsed_results = + results.each_with_object({}) do |(key, value), hash| + hex_key = "0x#{key.unpack1('H*')}" + hash[hex_key] = value + end.sort_by { |_k, v| -v } + data = + parsed_results.map do |result| + { Contract.where(is_lock_script: true).where.not(name: nil).where("type_hash = ? OR data_hash = ?", result[0], result[0]).first&.name => result[1] } + end + nil_sum = data.select { |item| item.keys.include?(nil) }.sum { |item| item[nil] } + filtered_data = data.reject { |item| item.keys.include?(nil) } + filtered_data << { "Others" => nil_sum } if nil_sum > 0 + filtered_data end private @@ -534,47 +552,48 @@ def aggron_first_day? # # Table name: daily_statistics # -# id :bigint not null, primary key -# transactions_count :string default("0") -# addresses_count :string default("0") -# total_dao_deposit :string default("0.0") -# block_timestamp :decimal(30, ) -# created_at_unixtimestamp :integer -# created_at :datetime not null -# updated_at :datetime not null -# dao_depositors_count :string default("0") -# unclaimed_compensation :string default("0") -# claimed_compensation :string default("0") -# average_deposit_time :string default("0") -# estimated_apc :string default("0") -# mining_reward :string default("0") -# deposit_compensation :string default("0") -# treasury_amount :string default("0") -# live_cells_count :string default("0") -# dead_cells_count :string default("0") -# avg_hash_rate :string default("0") -# avg_difficulty :string default("0") -# uncle_rate :string default("0") -# total_depositors_count :string default("0") -# total_tx_fee :decimal(30, ) -# address_balance_distribution :jsonb -# occupied_capacity :decimal(30, ) -# daily_dao_deposit :decimal(30, ) -# daily_dao_depositors_count :integer -# daily_dao_withdraw :decimal(30, ) -# circulation_ratio :decimal(, ) -# total_supply :decimal(30, ) -# circulating_supply :decimal(, ) -# block_time_distribution :jsonb -# epoch_time_distribution :jsonb -# epoch_length_distribution :jsonb -# average_block_time :jsonb -# nodes_distribution :jsonb -# nodes_count :integer -# locked_capacity :decimal(30, ) -# ckb_hodl_wave :jsonb -# holder_count :integer -# knowledge_size :decimal(30, ) +# id :bigint not null, primary key +# transactions_count :string default("0") +# addresses_count :string default("0") +# total_dao_deposit :string default("0.0") +# block_timestamp :decimal(30, ) +# created_at_unixtimestamp :integer +# created_at :datetime not null +# updated_at :datetime not null +# dao_depositors_count :string default("0") +# unclaimed_compensation :string default("0") +# claimed_compensation :string default("0") +# average_deposit_time :string default("0") +# estimated_apc :string default("0") +# mining_reward :string default("0") +# deposit_compensation :string default("0") +# treasury_amount :string default("0") +# live_cells_count :string default("0") +# dead_cells_count :string default("0") +# avg_hash_rate :string default("0") +# avg_difficulty :string default("0") +# uncle_rate :string default("0") +# total_depositors_count :string default("0") +# total_tx_fee :decimal(30, ) +# address_balance_distribution :jsonb +# occupied_capacity :decimal(30, ) +# daily_dao_deposit :decimal(30, ) +# daily_dao_depositors_count :integer +# daily_dao_withdraw :decimal(30, ) +# circulation_ratio :decimal(, ) +# total_supply :decimal(30, ) +# circulating_supply :decimal(, ) +# block_time_distribution :jsonb +# epoch_time_distribution :jsonb +# epoch_length_distribution :jsonb +# average_block_time :jsonb +# nodes_distribution :jsonb +# nodes_count :integer +# locked_capacity :decimal(30, ) +# ckb_hodl_wave :jsonb +# holder_count :integer +# knowledge_size :decimal(30, ) +# activity_address_contract_distribution :jsonb # # Indexes # diff --git a/app/models/rgbpp_assets_statistic.rb b/app/models/rgbpp_assets_statistic.rb new file mode 100644 index 000000000..71b5249f9 --- /dev/null +++ b/app/models/rgbpp_assets_statistic.rb @@ -0,0 +1,21 @@ +class RgbppAssetsStatistic < ApplicationRecord + enum :network, %i[global ckb btc] + enum :indicator, %i[ft_count dob_count holders_count transactions_count] +end + +# == Schema Information +# +# Table name: rgbpp_assets_statistics +# +# id :bigint not null, primary key +# indicator :integer not null +# value :decimal(40, ) default(0) +# network :integer default("global") +# created_at_unixtimestamp :integer +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# index_on_indicator_and_network_and_created_at_unixtimestamp (indicator,network,created_at_unixtimestamp) UNIQUE +# diff --git a/app/models/rgbpp_hourly_statistic.rb b/app/models/rgbpp_hourly_statistic.rb new file mode 100644 index 000000000..14c6f0681 --- /dev/null +++ b/app/models/rgbpp_hourly_statistic.rb @@ -0,0 +1,18 @@ +class RgbppHourlyStatistic < ApplicationRecord +end + +# == Schema Information +# +# Table name: rgbpp_hourly_statistics +# +# id :bigint not null, primary key +# xudt_count :integer default(0) +# dob_count :integer default(0) +# created_at_unixtimestamp :integer +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# index_rgbpp_hourly_statistics_on_created_at_unixtimestamp (created_at_unixtimestamp) UNIQUE +# diff --git a/app/models/udt_hourly_statistic.rb b/app/models/udt_hourly_statistic.rb new file mode 100644 index 000000000..233f2bb22 --- /dev/null +++ b/app/models/udt_hourly_statistic.rb @@ -0,0 +1,40 @@ +class UdtHourlyStatistic < ApplicationRecord + belongs_to :udt + + def percentage_change(attribute) + yesterday = previous_stat(udt_id, 1) + day_before_yesterday = previous_stat(udt_id, 2) + + return nil unless yesterday && day_before_yesterday + + yesterday_value = yesterday.public_send(attribute) + day_before_yesterday_value = day_before_yesterday.public_send(attribute) + + return nil if day_before_yesterday_value.zero? + + ((yesterday_value - day_before_yesterday_value).to_f / day_before_yesterday_value * 100).round(2) + end + + def previous_stat(udt_id, days_ago) + timestamp = (Time.current - days_ago.days).beginning_of_day.to_i + self.class.find_by(udt_id:, created_at_unixtimestamp: timestamp) + end +end + +# == Schema Information +# +# Table name: udt_hourly_statistics +# +# id :bigint not null, primary key +# udt_id :bigint not null +# ckb_transactions_count :integer default(0) +# amount :decimal(40, ) default(0) +# holders_count :integer default(0) +# created_at_unixtimestamp :integer +# created_at :datetime not null +# updated_at :datetime not null +# +# Indexes +# +# index_on_udt_id_and_unixtimestamp (udt_id,created_at_unixtimestamp) UNIQUE +# diff --git a/app/serializers/daily_statistic_serializer.rb b/app/serializers/daily_statistic_serializer.rb index dbd75c091..a03aed58b 100644 --- a/app/serializers/daily_statistic_serializer.rb +++ b/app/serializers/daily_statistic_serializer.rb @@ -138,4 +138,8 @@ class DailyStatisticSerializer attribute :knowledge_size, if: Proc.new { |_record, params| params.present? && params[:indicator].include?("knowledge_size") } + + attribute :activity_address_contract_distribution, if: Proc.new { |_record, params| + params.present? && params[:indicator].include?("activity_address_contract_distribution") + } end diff --git a/app/services/charts/daily_statistic_generator.rb b/app/services/charts/daily_statistic_generator.rb index 0d68f26e0..0043dd275 100644 --- a/app/services/charts/daily_statistic_generator.rb +++ b/app/services/charts/daily_statistic_generator.rb @@ -36,7 +36,7 @@ def updated_attrs treasury_amount estimated_apc live_cells_count dead_cells_count avg_hash_rate avg_difficulty uncle_rate address_balance_distribution total_tx_fee occupied_capacity daily_dao_deposit total_supply block_time_distribution - epoch_time_distribution epoch_length_distribution locked_capacity ckb_hodl_wave holder_count + epoch_time_distribution epoch_length_distribution locked_capacity ckb_hodl_wave holder_count activity_address_contract_distribution } established_order + others diff --git a/app/views/api/v2/fiber/graph_nodes/show.jbuilder b/app/views/api/v2/fiber/graph_nodes/show.jbuilder index d0dadbc14..2cc5aeb58 100644 --- a/app/views/api/v2/fiber/graph_nodes/show.jbuilder +++ b/app/views/api/v2/fiber/graph_nodes/show.jbuilder @@ -6,7 +6,7 @@ json.data do json.udt_cfg_infos @node.udt_cfg_infos json.fiber_graph_channels @graph_channels do |channel| - json.(channel, :channel_outpoint, :node1, :node2, :chain_hash, :open_transaction_info, :closed_transaction_info) + json.(channel, :channel_outpoint, :node1, :node2, :chain_hash, :open_transaction_info, :closed_transaction_info, :udt_info) json.funding_tx_block_number channel.funding_tx_block_number.to_s json.funding_tx_index channel.funding_tx_index.to_s json.last_updated_timestamp channel.last_updated_timestamp.to_s diff --git a/app/workers/fiber_graph_detect_worker.rb b/app/workers/fiber_graph_detect_worker.rb index b729b88fe..a39d9fd7f 100644 --- a/app/workers/fiber_graph_detect_worker.rb +++ b/app/workers/fiber_graph_detect_worker.rb @@ -2,9 +2,18 @@ class FiberGraphDetectWorker include Sidekiq::Worker sidekiq_options queue: "fiber" + attr_accessor :graph_node_ids, :graph_channel_outpoint + def perform + @graph_node_ids = [] + @graph_channel_outpoints = [] + # sync graph nodes and channels ["nodes", "channels"].each { fetch_graph_infos(_1) } + # purge outdated graph nodes + FiberGraphNode.where.not(node_id: @graph_node_ids).destroy_all + # purge outdated graph channels + FiberGraphChannel.where.not(channel_outpoint: @graph_channel_outpoints).destroy_all # check channel is closed FiberGraphChannel.open_channels.each do |channel| @@ -61,7 +70,7 @@ def upsert_node_with_cfg_info(node) peer_id: extract_peer_id(node["addresses"]), auto_accept_min_ckb_funding_amount: node["auto_accept_min_ckb_funding_amount"], } - + @graph_node_ids << node_attributes[:node_id] fiber_graph_node = FiberGraphNode.upsert(node_attributes, unique_by: %i[node_id], returning: %i[id]) return unless fiber_graph_node && node["udt_cfg_infos"].present? @@ -87,6 +96,7 @@ def build_channel_attributes(channel) channel_outpoint = channel["channel_outpoint"] open_transaction = CkbTransaction.find_by(tx_hash: channel_outpoint[0..65]) + @graph_channel_outpoints << channel_outpoint { channel_outpoint:, diff --git a/app/workers/generate_rgbpp_assets_statistic_worker.rb b/app/workers/generate_rgbpp_assets_statistic_worker.rb new file mode 100644 index 000000000..1eaa7978a --- /dev/null +++ b/app/workers/generate_rgbpp_assets_statistic_worker.rb @@ -0,0 +1,97 @@ +class GenerateRgbppAssetsStatisticWorker + include Sidekiq::Job + sidekiq_options queue: "rgbpp" + + attr_accessor :datetime + + def perform(datetime = nil) + @datetime = datetime + statistic_attributes = [ + ft_count_attributes, + dob_count_attributes, + btc_transactions_count_attributes, + ckb_transactions_count_attributes, + btc_holders_count_attributes, + ckb_holders_count_attributes, + ] + statistic_attributes.each { _1[:created_at_unixtimestamp] = started_at.to_i } + RgbppAssetsStatistic.upsert_all(statistic_attributes, unique_by: %i[indicator network created_at_unixtimestamp]) + rescue StandardError => e + Rails.logger.error "Error occurred during GenerateRgbppHourlyStatistic error: #{e.message}" + end + + private + + def ft_count_attributes + timestamp = CkbUtils.time_in_milliseconds(ended_at) - 1 + xudts_count = Udt.where(udt_type: %i[xudt xudt_compatible], block_timestamp: ..timestamp).count + { indicator: "ft_count", value: xudts_count, network: "global" } + end + + def dob_count_attributes + timestamp = CkbUtils.time_in_milliseconds(ended_at) - 1 + token_collections_count = TokenCollection.spore.where(block_timestamp: ..timestamp).count + { indicator: "dob_count", value: token_collections_count, network: "global" } + end + + def btc_transactions_count_attributes + transactions_count = BitcoinTransaction.where(time: ..ended_at.to_i).count + { indicator: "transactions_count", value: transactions_count, network: "btc" } + end + + def ckb_transactions_count_attributes + timestamp = CkbUtils.time_in_milliseconds(ended_at) - 1 + + ft_transaction_ids = Set.new + Udt.where(udt_type: %i[xudt xudt_compatible]).find_each do |xudt| + ft_transaction_ids.merge(xudt.ckb_transactions.where(block_timestamp: ..timestamp).ids) + end + + dob_transaction_ids = Set.new + TokenCollection.spore.find_each do |token_collection| + transfers = token_collection.transfers.joins(:ckb_transaction).where("ckb_transactions.block_timestamp <= ?", timestamp) + dob_transaction_ids.merge(transfers.map(&:transaction_id)) + end + + transactions_count = ft_transaction_ids.length + dob_transaction_ids.length + { indicator: "transactions_count", value: transactions_count, network: "ckb" } + end + + def btc_holders_count_attributes + udt_types = %i[xudt xudt_compatible spore_cell did_cell] + udt_ids = Udt.where(udt_type: udt_types, published: true).ids + address_ids = UdtAccount.where(udt_id: udt_ids).where("amount > 0").pluck(:address_id).uniq + holders_count = BitcoinAddressMapping.where(ckb_address_id: address_ids, created_at: ..ended_at). + distinct.count(:bitcoin_address_id) + { indicator: "holders_count", value: holders_count, network: "btc" } + end + + def ckb_holders_count_attributes + udt_types = %i[xudt xudt_compatible spore_cell did_cell] + udt_ids = Udt.where(udt_type: udt_types, published: true).ids + holders_count = UdtAccount.where(udt_id: udt_ids, created_at: ..ended_at). + where("amount > 0").distinct.count(:address_id) + { indicator: "holders_count", value: holders_count, network: "ckb" } + end + + def to_be_counted_date + if @datetime.present? + return Time.parse(@datetime) + end + + last_record = UdtHourlyStatistic.order(created_at_unixtimestamp: :desc).first + if last_record + Time.at(last_record.created_at_unixtimestamp) + 1.day + else + Time.now.yesterday + end + end + + def started_at + @started_at ||= to_be_counted_date.beginning_of_day + end + + def ended_at + @ended_at ||= to_be_counted_date.end_of_day + end +end diff --git a/app/workers/generate_udt_hourly_statistic_worker.rb b/app/workers/generate_udt_hourly_statistic_worker.rb new file mode 100644 index 000000000..74c9804a5 --- /dev/null +++ b/app/workers/generate_udt_hourly_statistic_worker.rb @@ -0,0 +1,61 @@ +class GenerateUdtHourlyStatisticWorker + include Sidekiq::Job + + def perform(datetime = nil) + ActiveRecord::Base.connection.execute("SET statement_timeout = 0") + start_time = to_be_counted_date(datetime) + generate_statistics(start_time) + ActiveRecord::Base.connection.execute("RESET statement_timeout") + rescue StandardError => e + Rails.logger.error "Error occurred during GenerateUdtHourlyStatistic error: #{e.message}" + end + + private + + def to_be_counted_date(datetime) + last_record = UdtHourlyStatistic.order(created_at_unixtimestamp: :desc).first + if last_record + Time.zone.at(last_record.created_at_unixtimestamp) + 1.day + else + datetime.is_a?(String) ? Time.zone.parse(datetime) : Time.current.yesterday + end + end + + def generate_statistics(start_time) + puts "Generating udt hourly statistics for #{start_time}" + statistic_attributes = [] + udt_types = %i[xudt xudt_compatible spore_cell did_cell] + Udt.where(udt_type: udt_types, published: true).find_each do |udt| + statistic_attributes << { + udt_id: udt.id, + amount: calc_amount(udt), + ckb_transactions_count: calc_ckb_transactions_count(udt), + holders_count: calc_holders_count(udt), + created_at_unixtimestamp: start_time.beginning_of_day.to_i, + } + end + + if statistic_attributes.present? + UdtHourlyStatistic.upsert_all(statistic_attributes, unique_by: %i[udt_id created_at_unixtimestamp]) + end + end + + def calc_amount(udt) + inputs_amount = 0 + outputs_amount = 0 + udt.ckb_transactions.includes(:cell_outputs).find_in_batches(batch_size: 1000) do |transactions| + ids = transactions.map(&:id) + inputs_amount += CellOutput.select(:udt_amount).where(consumed_by_id: ids).sum(:udt_amount) + outputs_amount += CellOutput.select(:udt_amount).where(ckb_transaction_id: ids).sum(:udt_amount) + end + [inputs_amount, outputs_amount].max + end + + def calc_ckb_transactions_count(udt) + udt.ckb_transactions.count + end + + def calc_holders_count(udt) + udt.udt_holder_allocations.sum("ckb_holder_count + btc_holder_count") + end +end diff --git a/config/routes/v2.rb b/config/routes/v2.rb index 6a74404f9..a72b59b95 100644 --- a/config/routes/v2.rb +++ b/config/routes/v2.rb @@ -106,5 +106,7 @@ resources :graph_nodes, param: :node_id, only: %i[index show] resources :graph_channels, only: :index end + resources :udt_hourly_statistics, only: :show + resources :rgbpp_assets_statistics, only: :index end end diff --git a/config/settings.mainnet.yml b/config/settings.mainnet.yml index b68e08602..bbe41057b 100644 --- a/config/settings.mainnet.yml +++ b/config/settings.mainnet.yml @@ -84,6 +84,7 @@ type_id_code_hash: "0x0000000000000000000000000000000000000000000000000054595045 homepage_transactions_records_count: 15 homepage_block_records_count: 15 proposal_window: 10 +query_default_limit: 5000 # rgbpp code hash rgbpp_code_hash: diff --git a/config/settings.testnet.yml b/config/settings.testnet.yml index 790e071cd..707f2faac 100644 --- a/config/settings.testnet.yml +++ b/config/settings.testnet.yml @@ -84,6 +84,7 @@ type_id_code_hash: "0x0000000000000000000000000000000000000000000000000054595045 homepage_transactions_records_count: 15 homepage_block_records_count: 15 proposal_window: 10 +query_default_limit: 5000 # rgbpp code hash rgbpp_code_hash: diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 7054f09af..ae1f6b56e 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -7,6 +7,7 @@ - bitcoin - fiber - low + - rgbpp production: :concurrency: 20 diff --git a/db/migrate/20241212022531_create_udt_hourly_statistics.rb b/db/migrate/20241212022531_create_udt_hourly_statistics.rb new file mode 100644 index 000000000..4cb58413f --- /dev/null +++ b/db/migrate/20241212022531_create_udt_hourly_statistics.rb @@ -0,0 +1,14 @@ +class CreateUdtHourlyStatistics < ActiveRecord::Migration[7.0] + def change + create_table :udt_hourly_statistics do |t| + t.bigint :udt_id, null: false + t.integer :ckb_transactions_count, default: 0 + t.decimal :amount, precision: 40, default: 0.0 + t.integer :holders_count, default: 0 + t.integer :created_at_unixtimestamp + t.timestamps + end + + add_index :udt_hourly_statistics, %i[udt_id created_at_unixtimestamp], name: "index_on_udt_id_and_unixtimestamp", unique: true + end +end diff --git a/db/migrate/20241213053309_create_rgbpp_hourly_statistics.rb b/db/migrate/20241213053309_create_rgbpp_hourly_statistics.rb new file mode 100644 index 000000000..1b12c0c42 --- /dev/null +++ b/db/migrate/20241213053309_create_rgbpp_hourly_statistics.rb @@ -0,0 +1,12 @@ +class CreateRgbppHourlyStatistics < ActiveRecord::Migration[7.0] + def change + create_table :rgbpp_hourly_statistics do |t| + t.integer :xudt_count, default: 0 + t.integer :dob_count, default: 0 + t.integer :created_at_unixtimestamp + t.timestamps + end + + add_index :rgbpp_hourly_statistics, :created_at_unixtimestamp, unique: true + end +end diff --git a/db/migrate/20241218085721_create_rgbpp_assets_statistics.rb b/db/migrate/20241218085721_create_rgbpp_assets_statistics.rb new file mode 100644 index 000000000..a14b32422 --- /dev/null +++ b/db/migrate/20241218085721_create_rgbpp_assets_statistics.rb @@ -0,0 +1,15 @@ +class CreateRgbppAssetsStatistics < ActiveRecord::Migration[7.0] + def change + create_table :rgbpp_assets_statistics do |t| + t.integer :indicator, null: false + t.decimal :value, precision: 40, default: 0.0 + t.integer :network, default: 0 + t.integer :created_at_unixtimestamp + + t.timestamps + end + + add_index :rgbpp_assets_statistics, %i[indicator network created_at_unixtimestamp], unique: true, + name: "index_on_indicator_and_network_and_created_at_unixtimestamp" + end +end diff --git a/db/migrate/20241225045757_add_activity_address_contract_distribution_to_daily_statistic.rb b/db/migrate/20241225045757_add_activity_address_contract_distribution_to_daily_statistic.rb new file mode 100644 index 000000000..e09e52866 --- /dev/null +++ b/db/migrate/20241225045757_add_activity_address_contract_distribution_to_daily_statistic.rb @@ -0,0 +1,5 @@ +class AddActivityAddressContractDistributionToDailyStatistic < ActiveRecord::Migration[7.0] + def change + add_column :daily_statistics, :activity_address_contract_distribution, :jsonb + end +end diff --git a/db/structure.sql b/db/structure.sql index c1f18c9b6..a8e7a0280 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1054,12 +1054,12 @@ ALTER SEQUENCE public.cell_data_cell_output_id_seq OWNED BY public.cell_data.cel CREATE TABLE public.cell_dependencies ( id bigint NOT NULL, + contract_id bigint, ckb_transaction_id bigint NOT NULL, dep_type integer, contract_cell_id bigint NOT NULL, script_id bigint, - contract_id bigint, - implicit boolean, + implicit boolean DEFAULT true NOT NULL, block_number bigint, tx_index integer, contract_analyzed boolean DEFAULT false @@ -1574,7 +1574,8 @@ CREATE TABLE public.daily_statistics ( locked_capacity numeric(30,0), ckb_hodl_wave jsonb, holder_count integer, - knowledge_size numeric(30,0) + knowledge_size numeric(30,0), + activity_address_contract_distribution jsonb ); @@ -2344,6 +2345,73 @@ CREATE SEQUENCE public.reject_reasons_id_seq ALTER SEQUENCE public.reject_reasons_id_seq OWNED BY public.reject_reasons.id; +-- +-- Name: rgbpp_assets_statistics; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.rgbpp_assets_statistics ( + id bigint NOT NULL, + indicator integer NOT NULL, + value numeric(40,0) DEFAULT 0.0, + network integer DEFAULT 0, + created_at_unixtimestamp integer, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + +-- +-- Name: rgbpp_assets_statistics_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.rgbpp_assets_statistics_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: rgbpp_assets_statistics_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.rgbpp_assets_statistics_id_seq OWNED BY public.rgbpp_assets_statistics.id; + + +-- +-- Name: rgbpp_hourly_statistics; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.rgbpp_hourly_statistics ( + id bigint NOT NULL, + xudt_count integer DEFAULT 0, + dob_count integer DEFAULT 0, + created_at_unixtimestamp integer, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + +-- +-- Name: rgbpp_hourly_statistics_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.rgbpp_hourly_statistics_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: rgbpp_hourly_statistics_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.rgbpp_hourly_statistics_id_seq OWNED BY public.rgbpp_hourly_statistics.id; + + -- -- Name: rolling_avg_block_time; Type: MATERIALIZED VIEW; Schema: public; Owner: - -- @@ -2698,6 +2766,41 @@ CREATE SEQUENCE public.udt_holder_allocations_id_seq ALTER SEQUENCE public.udt_holder_allocations_id_seq OWNED BY public.udt_holder_allocations.id; +-- +-- Name: udt_hourly_statistics; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.udt_hourly_statistics ( + id bigint NOT NULL, + udt_id bigint NOT NULL, + ckb_transactions_count integer DEFAULT 0, + amount numeric(40,0) DEFAULT 0.0, + holders_count integer DEFAULT 0, + created_at_unixtimestamp integer, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + +-- +-- Name: udt_hourly_statistics_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.udt_hourly_statistics_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: udt_hourly_statistics_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.udt_hourly_statistics_id_seq OWNED BY public.udt_hourly_statistics.id; + + -- -- Name: udt_transactions; Type: TABLE; Schema: public; Owner: - -- @@ -3301,6 +3404,20 @@ ALTER TABLE ONLY public.portfolios ALTER COLUMN id SET DEFAULT nextval('public.p ALTER TABLE ONLY public.reject_reasons ALTER COLUMN id SET DEFAULT nextval('public.reject_reasons_id_seq'::regclass); +-- +-- Name: rgbpp_assets_statistics id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.rgbpp_assets_statistics ALTER COLUMN id SET DEFAULT nextval('public.rgbpp_assets_statistics_id_seq'::regclass); + + +-- +-- Name: rgbpp_hourly_statistics id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.rgbpp_hourly_statistics ALTER COLUMN id SET DEFAULT nextval('public.rgbpp_hourly_statistics_id_seq'::regclass); + + -- -- Name: statistic_infos id; Type: DEFAULT; Schema: public; Owner: - -- @@ -3364,6 +3481,13 @@ ALTER TABLE ONLY public.udt_accounts ALTER COLUMN id SET DEFAULT nextval('public ALTER TABLE ONLY public.udt_holder_allocations ALTER COLUMN id SET DEFAULT nextval('public.udt_holder_allocations_id_seq'::regclass); +-- +-- Name: udt_hourly_statistics id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.udt_hourly_statistics ALTER COLUMN id SET DEFAULT nextval('public.udt_hourly_statistics_id_seq'::regclass); + + -- -- Name: udt_verifications id; Type: DEFAULT; Schema: public; Owner: - -- @@ -3870,6 +3994,22 @@ ALTER TABLE ONLY public.reject_reasons ADD CONSTRAINT reject_reasons_pkey PRIMARY KEY (id); +-- +-- Name: rgbpp_assets_statistics rgbpp_assets_statistics_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.rgbpp_assets_statistics + ADD CONSTRAINT rgbpp_assets_statistics_pkey PRIMARY KEY (id); + + +-- +-- Name: rgbpp_hourly_statistics rgbpp_hourly_statistics_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.rgbpp_hourly_statistics + ADD CONSTRAINT rgbpp_hourly_statistics_pkey PRIMARY KEY (id); + + -- -- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -3950,6 +4090,14 @@ ALTER TABLE ONLY public.udt_holder_allocations ADD CONSTRAINT udt_holder_allocations_pkey PRIMARY KEY (id); +-- +-- Name: udt_hourly_statistics udt_hourly_statistics_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.udt_hourly_statistics + ADD CONSTRAINT udt_hourly_statistics_pkey PRIMARY KEY (id); + + -- -- Name: udt_verifications udt_verifications_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -4792,6 +4940,20 @@ CREATE INDEX index_cell_dependencies_on_block_number_and_tx_index ON public.cell CREATE INDEX index_cell_dependencies_on_contract_analyzed ON public.cell_dependencies USING btree (contract_analyzed); +-- +-- Name: index_cell_dependencies_on_contract_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_cell_dependencies_on_contract_id ON public.cell_dependencies USING btree (contract_id); + + +-- +-- Name: index_cell_dependencies_on_script_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_cell_dependencies_on_script_id ON public.cell_dependencies USING btree (script_id); + + -- -- Name: index_cell_dependencies_on_tx_id_and_cell_id_and_dep_type; Type: INDEX; Schema: public; Owner: - -- @@ -5044,6 +5206,20 @@ CREATE UNIQUE INDEX index_omiga_inscription_infos_on_udt_hash ON public.omiga_in CREATE INDEX index_on_cell_dependencies_contract_cell_block_tx ON public.cell_dependencies USING btree (contract_cell_id, block_number DESC, tx_index DESC); +-- +-- Name: index_on_indicator_and_network_and_created_at_unixtimestamp; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_on_indicator_and_network_and_created_at_unixtimestamp ON public.rgbpp_assets_statistics USING btree (indicator, network, created_at_unixtimestamp); + + +-- +-- Name: index_on_udt_id_and_unixtimestamp; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_on_udt_id_and_unixtimestamp ON public.udt_hourly_statistics USING btree (udt_id, created_at_unixtimestamp); + + -- -- Name: index_portfolios_on_user_id_and_address_id; Type: INDEX; Schema: public; Owner: - -- @@ -5058,6 +5234,13 @@ CREATE UNIQUE INDEX index_portfolios_on_user_id_and_address_id ON public.portfol CREATE UNIQUE INDEX index_reject_reasons_on_ckb_transaction_id ON public.reject_reasons USING btree (ckb_transaction_id); +-- +-- Name: index_rgbpp_hourly_statistics_on_created_at_unixtimestamp; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_rgbpp_hourly_statistics_on_created_at_unixtimestamp ON public.rgbpp_hourly_statistics USING btree (created_at_unixtimestamp); + + -- -- Name: index_rolling_avg_block_time_on_timestamp; Type: INDEX; Schema: public; Owner: - -- @@ -6127,6 +6310,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20240918033146'), ('20240920094807'), ('20240924065539'), +('20241009081935'), ('20241012014906'), ('20241023055256'), ('20241023063536'), @@ -6141,6 +6325,10 @@ INSERT INTO "schema_migrations" (version) VALUES ('20241129000339'), ('20241129032447'), ('20241202072604'), -('20241205023729'); +('20241205023729'), +('20241212022531'), +('20241213053309'), +('20241218085721'), +('20241225045757'); diff --git a/lib/scheduler.rb b/lib/scheduler.rb index 9afe16d6f..b9006f311 100644 --- a/lib/scheduler.rb +++ b/lib/scheduler.rb @@ -32,6 +32,8 @@ def call_worker(clz) s.cron "0 8 * * *" do call_worker Charts::DailyStatistic + call_worker GenerateUdtHourlyStatisticWorker + call_worker GenerateRgbppAssetsStatisticWorker end s.every "10m", overlap: false do diff --git a/test/controllers/api/v1/address_live_cells_controller_test.rb b/test/controllers/api/v1/address_live_cells_controller_test.rb index a4446429c..91d6eaf57 100644 --- a/test/controllers/api/v1/address_live_cells_controller_test.rb +++ b/test/controllers/api/v1/address_live_cells_controller_test.rb @@ -23,7 +23,7 @@ class AddressLiveCellsControllerTest < ActionDispatch::IntegrationTest address = create(:address, :with_udt_transactions) valid_get api_v1_address_live_cell_url(address.address_hash) - assert_equal ({ "data" => [], "meta" => { "total" => 0, "page_size" => 20 } }), json + assert_equal ({ "data" => [], "meta" => { "total" => 0, "page_size" => 20, "total_pages" => 0 } }), json end test "should return all live cells" do