diff --git a/backends/README.md b/backends/README.md new file mode 100644 index 000000000..2eb15d626 --- /dev/null +++ b/backends/README.md @@ -0,0 +1,33 @@ +# Backends + +## `meta_parameters_struct` and `meta_parameters_function` + +`_meta_parameters.yaml` can carry two more sections of the same kind +as `meta_parameters` itself: facts the header knows that the C declaration does +not carry. These two say how to print bytes whose meaning no C type states -- +`meta_parameters_struct` for a struct's byte-array members, +`meta_parameters_function` for a function's byte-array parameters. + +```yaml +meta_parameters_struct: + ze_uuid_t: + - [ uuid_reversed, id ] + ze_kernel_uuid_t: + - [ uuid_reversed, kid ] + - [ uuid_reversed, mid ] + ze_ipc_mem_handle_t: + - [ blob, data ] + +meta_parameters_function: + cuDeviceGetLuid: + - [ uuid, luid ] +``` + +A member with no row falls back to the default print: `char[N]` prints as a C +string, stopping at the first null char. + +| Renderer | Prints | +| --- | --- | +| `uuid` | dashed hex, first byte first (cuda, hip) | +| `uuid_reversed` | dashed hex, last byte first (ze, zes) | +| `blob` | every byte escaped, stopping at none | diff --git a/backends/cuda/cuda_meta_parameters.yaml b/backends/cuda/cuda_meta_parameters.yaml index d2f083804..19622a41d 100644 --- a/backends/cuda/cuda_meta_parameters.yaml +++ b/backends/cuda/cuda_meta_parameters.yaml @@ -1078,3 +1078,19 @@ meta_parameters: - [InArray, globalDim, tensorRank] - [InArray, globalStrides, tensorRank] - [InArray, elementStrides, tensorRank] + +meta_parameters_function: + cuDeviceGetLuid: + - [ uuid, luid ] + +meta_parameters_struct: + CUuuid: + - [ uuid, bytes ] + CUmemFabricHandle_v1: + - [ blob, data ] + CUipcEventHandle_v1: + - [ blob, reserved ] + CUipcMemHandle_v1: + - [ blob, reserved ] + CUmemPoolPtrExportData_v1: + - [ blob, reserved ] diff --git a/backends/cuda/cuda_model.rb b/backends/cuda/cuda_model.rb index 351199464..a0f92290e 100644 --- a/backends/cuda/cuda_model.rb +++ b/backends/cuda/cuda_model.rb @@ -14,10 +14,12 @@ # The driver and its export tables are one API but two LTTng providers, so the # commands are grouped by the provider that will carry them. +META_PARAMETERS = load_meta_parameters('cuda_meta_parameters.yaml', 'cuda_exports_meta_parameters.yaml') + COMMANDS = build_command_index( { lttng_ust_cuda: cuda_api.functions, lttng_ust_cuda_exports: cuda_exports_api.functions }, context: CONTEXT, - spec: load_meta_parameters('cuda_meta_parameters.yaml', 'cuda_exports_meta_parameters.yaml') + spec: META_PARAMETERS[:meta_parameters] ) CUDA_POINTER_NAMES = COMMANDS.pointer_names diff --git a/backends/cuda/gen_babeltrace_cuda_lib.rb b/backends/cuda/gen_babeltrace_cuda_lib.rb index 6577840b2..cc99beae3 100644 --- a/backends/cuda/gen_babeltrace_cuda_lib.rb +++ b/backends/cuda/gen_babeltrace_cuda_lib.rb @@ -1,4 +1,4 @@ require_relative 'gen_cuda_library_base' require_relative '../../utils/gen_babeltrace_lib_helper' -print_babeltrace_lib(NAMING) +print_babeltrace_lib(NAMING, META_PARAMETERS) diff --git a/backends/cuda/gen_cuda_library.rb b/backends/cuda/gen_cuda_library.rb index c65c1c887..eb5cfdf19 100644 --- a/backends/cuda/gen_cuda_library.rb +++ b/backends/cuda/gen_cuda_library.rb @@ -30,8 +30,7 @@ module CUDA EOF -print_handle_uuid_modules -puts +print_bytes_module(NAMING, META_PARAMETERS) puts <(name, t) { print_enum(NAMING, name, API.enum(t.type, opaque_ok: true) || t.type) }, - struct: ->(name, t) { print_struct_prepending_uuid(NAMING, name, API.struct(t.type)) }, + struct: lambda { |name, t| + struct = API.struct(t.type) + print_struct_with_namespace(NAMING, name, struct, + body: struct_to_s_definition(NAMING, struct, META_PARAMETERS[:meta_parameters_struct][name])) + }, pointer: nil, integer: nil ) diff --git a/backends/hip/gen_babeltrace_hip_lib.rb b/backends/hip/gen_babeltrace_hip_lib.rb index 6653d3b9e..fc0264666 100644 --- a/backends/hip/gen_babeltrace_hip_lib.rb +++ b/backends/hip/gen_babeltrace_hip_lib.rb @@ -1,4 +1,4 @@ require_relative 'gen_hip_library_base' require_relative '../../utils/gen_babeltrace_lib_helper' -print_babeltrace_lib(NAMING) +print_babeltrace_lib(NAMING, META_PARAMETERS) diff --git a/backends/hip/gen_hip_library.rb b/backends/hip/gen_hip_library.rb index c7db3a3d1..865a64075 100644 --- a/backends/hip/gen_hip_library.rb +++ b/backends/hip/gen_hip_library.rb @@ -10,7 +10,7 @@ module HIP EOF -print_handle_uuid_modules +print_bytes_module(NAMING, META_PARAMETERS) puts <(func) { traced_functions.include?(func.name) } ) diff --git a/backends/mpi/gen_babeltrace_mpi_lib.rb b/backends/mpi/gen_babeltrace_mpi_lib.rb index 49a68924d..f10a02a2c 100644 --- a/backends/mpi/gen_babeltrace_mpi_lib.rb +++ b/backends/mpi/gen_babeltrace_mpi_lib.rb @@ -1,4 +1,4 @@ require_relative 'gen_mpi_library_base' require_relative '../../utils/gen_babeltrace_lib_helper' -print_babeltrace_lib(NAMING) +print_babeltrace_lib(NAMING, META_PARAMETERS) diff --git a/backends/mpi/gen_mpi_library.rb b/backends/mpi/gen_mpi_library.rb index 7dc370ae4..beb01fb86 100644 --- a/backends/mpi/gen_mpi_library.rb +++ b/backends/mpi/gen_mpi_library.rb @@ -8,7 +8,7 @@ module MPI EOF -print_handle_uuid_modules +print_bytes_module(NAMING, META_PARAMETERS) print_typedefs(NAMING) diff --git a/backends/mpi/mpi_model.rb b/backends/mpi/mpi_model.rb index f9e5c00cb..0d423505a 100644 --- a/backends/mpi/mpi_model.rb +++ b/backends/mpi/mpi_model.rb @@ -38,9 +38,11 @@ CONTEXT = BackendContext.for(API, result_name: 'mpiResult', init_functions: init_functions) +META_PARAMETERS = load_meta_parameters('mpi_meta_parameters.yaml') + COMMANDS = build_command_index( { lttng_ust_mpi: API.functions }, - context: CONTEXT, spec: load_meta_parameters('mpi_meta_parameters.yaml') + context: CONTEXT, spec: META_PARAMETERS[:meta_parameters] ) # MPI spells its functions MPI_Comm_rank, already snake_case, so the macro name diff --git a/backends/omp/gen_babeltrace_omp_lib.rb b/backends/omp/gen_babeltrace_omp_lib.rb index cba811e84..c8ecf5562 100644 --- a/backends/omp/gen_babeltrace_omp_lib.rb +++ b/backends/omp/gen_babeltrace_omp_lib.rb @@ -1,4 +1,4 @@ require_relative 'gen_omp_library_base' require_relative '../../utils/gen_babeltrace_lib_helper' -print_babeltrace_lib(NAMING) +print_babeltrace_lib(NAMING, META_PARAMETERS) diff --git a/backends/omp/gen_omp_library.rb b/backends/omp/gen_omp_library.rb index a2c2206d2..a2900ed9d 100644 --- a/backends/omp/gen_omp_library.rb +++ b/backends/omp/gen_omp_library.rb @@ -17,6 +17,8 @@ module OMP EOF +print_bytes_module(NAMING, META_PARAMETERS) + # The Ruby bindings for OMPT carry only its enums. print_typedefs(NAMING, enum: ->(name, t) { print_enum(name, API.enum(t.type)) }, diff --git a/backends/omp/ompt_model.rb b/backends/omp/ompt_model.rb index 6ea62fa31..eaffbaafa 100644 --- a/backends/omp/ompt_model.rb +++ b/backends/omp/ompt_model.rb @@ -14,7 +14,9 @@ YAMLCAst::Declaration.new(name: t.name.gsub(/_t\z/, '') + '_func', type: t.type.type) end +META_PARAMETERS = load_meta_parameters('ompt_meta_parameters.yaml') + COMMANDS = build_command_index( { lttng_ust_ompt: OMPT_CALLBACKS }, - context: CONTEXT, spec: load_meta_parameters('ompt_meta_parameters.yaml') + context: CONTEXT, spec: META_PARAMETERS[:meta_parameters] ) diff --git a/backends/opencl/opencl_model.rb b/backends/opencl/opencl_model.rb index 02a825bdc..c10737682 100644 --- a/backends/opencl/opencl_model.rb +++ b/backends/opencl/opencl_model.rb @@ -678,7 +678,7 @@ def void_parameters? end end -meta_parameters = load_meta_parameters('opencl_meta_parameters.yaml') +meta_parameters = load_meta_parameters('opencl_meta_parameters.yaml')[:meta_parameters] # Both groups go to the one lttng_ust_opencl provider, so they are grouped by # what actually separates them: an extension is reached through diff --git a/backends/ze/gen_babeltrace_ze_lib.rb b/backends/ze/gen_babeltrace_ze_lib.rb index f0a328244..520f50340 100644 --- a/backends/ze/gen_babeltrace_ze_lib.rb +++ b/backends/ze/gen_babeltrace_ze_lib.rb @@ -1,4 +1,4 @@ require_relative 'gen_ze_library_base' require_relative '../../utils/gen_babeltrace_lib_helper' -print_babeltrace_lib(NAMING) +print_babeltrace_lib(NAMING, META_PARAMETERS) diff --git a/backends/ze/gen_ze_library.rb b/backends/ze/gen_ze_library.rb index 269f8515a..0ef6f2c11 100644 --- a/backends/ze/gen_ze_library.rb +++ b/backends/ze/gen_ze_library.rb @@ -102,92 +102,6 @@ def self.ZE_MINOR_VERSION(ver = VERSION_CURRENT) ver & 0x0000ffff end - module Handle - def to_s - s = '{ data: "' - s << self[:data].to_a.collect { |v| "\\\\x%02x" % ((v + 256)%256) }.join - s << '" }' - end - end - - module UUID - def to_s - a = self[:id].to_a - s = "{ id: " - s << "%02x" % a[15] - s << "%02x" % a[14] - s << "%02x" % a[13] - s << "%02x" % a[12] - s << "-" - s << "%02x" % a[11] - s << "%02x" % a[10] - s << "-" - s << "%02x" % a[9] - s << "%02x" % a[8] - s << "-" - s << "%02x" % a[7] - s << "%02x" % a[6] - s << "-" - s << "%02x" % a[5] - s << "%02x" % a[4] - s << "%02x" % a[3] - s << "%02x" % a[2] - s << "%02x" % a[1] - s << "%02x" % a[0] - s << " }" - end - end - - module KUUID - def to_s - a = self[:kid].to_a - s = "{ kid: " - s << "%02x" % a[15] - s << "%02x" % a[14] - s << "%02x" % a[13] - s << "%02x" % a[12] - s << "-" - s << "%02x" % a[11] - s << "%02x" % a[10] - s << "-" - s << "%02x" % a[9] - s << "%02x" % a[8] - s << "-" - s << "%02x" % a[7] - s << "%02x" % a[6] - s << "-" - s << "%02x" % a[5] - s << "%02x" % a[4] - s << "%02x" % a[3] - s << "%02x" % a[2] - s << "%02x" % a[1] - s << "%02x" % a[0] - a = self[:mid].to_a - s << ", mid: " - s << "%02x" % a[15] - s << "%02x" % a[14] - s << "%02x" % a[13] - s << "%02x" % a[12] - s << "-" - s << "%02x" % a[11] - s << "%02x" % a[10] - s << "-" - s << "%02x" % a[9] - s << "%02x" % a[8] - s << "-" - s << "%02x" % a[7] - s << "%02x" % a[6] - s << "-" - s << "%02x" % a[5] - s << "%02x" % a[4] - s << "%02x" % a[3] - s << "%02x" % a[2] - s << "%02x" % a[1] - s << "%02x" % a[0] - s << " }" - end - end - module Version def to_s "\#{self[:major]}.\#{self[:minor]}" @@ -201,18 +115,9 @@ def to_i EOF -def print_struct(name, struct) - prepends = [] - if NAMING.class_name(name).match('UUID') - prepends << if NAMING.class_name(name).match('ZEKernelUUID') - 'KUUID' - else - 'UUID' - end - elsif NAMING.class_name(name).match(/Handle\z/) - prepends << 'Handle' - end +print_bytes_module(NAMING, META_PARAMETERS) +def print_struct(name, struct) stype = traced_structure_type_names(name).first initializer = <:_; a parameter is traced under `_val`, or +# `_vals` when the field holds several of them. +# +# >> event_function_name('lttng_ust_cuda:cuDeviceGetLuid_exit') +# => "cuDeviceGetLuid" +# >> parameter_name('luid_vals') +# => "luid" +def event_function_name(event_name) + event_name.split(':').last.sub(/_(#{START}|#{STOP})\z/, '') +end + +def parameter_name(field_name) + field_name.sub(/_vals?\z/, '') +end + # The tracepoint macro takes a fixed number of arguments, one of which LTTng # spends itself; a function with more parameters than the rest can carry has no # tracepoint generated for it. diff --git a/utils/command_index.rb b/utils/command_index.rb index 6595c2724..6d9c1709a 100644 --- a/utils/command_index.rb +++ b/utils/command_index.rb @@ -15,14 +15,11 @@ class CommandIndex def initialize(groups) @groups = groups.freeze - @by_name = {} - @groups.each_value do |commands| - commands.each do |c| - raise "#{c.name} appears in two command lists" if @by_name.key?(c.name) + all = @groups.values.flatten + twice = all.collect(&:name).tally.select { |_, n| n > 1 }.keys + raise "#{twice.join(', ')} appear in two command lists" unless twice.empty? - @by_name[c.name] = c - end - end + @by_name = all.to_h { |c| [c.name, c] } end def each(&block) diff --git a/utils/gen_babeltrace_lib_helper.rb b/utils/gen_babeltrace_lib_helper.rb index 42e6d9726..655b69a9c 100644 --- a/utils/gen_babeltrace_lib_helper.rb +++ b/utils/gen_babeltrace_lib_helper.rb @@ -3,37 +3,68 @@ # The whole of a backend's babeltrace-library generator: require the FFI # bindings, then emit one pretty-printer per event in the model. -def print_babeltrace_lib(naming) +def print_babeltrace_lib(naming, meta_parameters) puts "require_relative '#{naming.backend}_library.rb'" - add_babeltrace_event_callbacks(naming, "btx_#{naming.backend}_model.yaml") + add_babeltrace_event_callbacks(naming, "btx_#{naming.backend}_model.yaml", + meta_parameters[:meta_parameters_function]) +end + +def payload_fields(event) + event[:payload_field_class]&.[](:members) || [] end # One `$event_lambdas` entry per event: a lambda that renders the event's # payload as a string. -def add_babeltrace_event_callbacks(naming, file) - yaml_load_file_cached(file)[:stream_classes].each do |s| - s[:event_classes].each do |e| - # Handle payload_field_class not present, in this case empty array - members = e[:payload_field_class]&.[](:members).to_a - fields = members.reject { |f| length_field_name?(f[:name]) } - .map { |f| render_field(naming, f) } +def add_babeltrace_event_callbacks(naming, file, meta_parameters_function) + event_classes = yaml_load_file_cached(file)[:stream_classes].flat_map { |s| s[:event_classes] } + check_meta_parameters_function(meta_parameters_function, byte_array_parameters(event_classes)) + + event_classes.each do |e| + renderers = meta_parameters_function.fetch(event_function_name(e[:name]), {}) + fields = payload_fields(e).filter_map do |f| + next if length_field_name?(f[:name]) + + render_field(naming, f, renderers[parameter_name(f[:name])]) + end + + puts <<~EOF + $event_lambdas["#{e[:name]}"] = lambda { |defi| + s = "{ " + #{fields.join("\n s << ', '\n ")} + s << " }" + } + EOF + end +end + +# A byte array reaches the payload as a string, we will print it according to +# metadata. +def byte_array_parameters(event_classes) + event_classes.group_by { |e| event_function_name(e[:name]) }.transform_values do |events| + events.flat_map { |e| payload_fields(e) } + .select { |f| f[:field_class][:type] == 'string' } + .collect { |f| parameter_name(f[:name]) } + end +end - # Now just print the full strings to pretty printf the struct - puts <<~EOF - $event_lambdas["#{e[:name]}"] = lambda { |defi| - s = "{ " - #{fields.join("\n s << ', '\n ")} - s << " }" - } - EOF +def check_meta_parameters_function(meta_parameters_function, byte_arrays) + meta_parameters_function.each do |function, parameters| + bytes = byte_arrays.fetch(function) do + raise "meta_parameters_function names no traced function: #{function}" end + unrenderable = parameters.keys - bytes + next if unrenderable.empty? + + raise "#{function} traces no byte-array parameter #{unrenderable.join(', ')} " \ + "(traces #{bytes.empty? ? 'none' : bytes.join(', ')})" end end # The statement that appends one field to the rendered payload. `be_class` is # the FFI class for a field whose raw bytes mean something richer -- an enum, a -# bitmask, a struct -- and is absent for one that prints as itself. -def render_field(naming, field) +# bitmask, a struct -- and is absent for one that prints as itself. `renderer` +# is the `Bytes` function declared for bytes that mean something no type says. +def render_field(naming, field, renderer = nil) name = field[:name] fc = field[:field_class] be_class = field[:metadata]&.[](:be_class) @@ -51,7 +82,13 @@ def render_field(naming, field) when 'double', 'single' plain when 'string' - be_class ? render_packed_struct(name, be_class) : %(s << "#{name}: \#{defi["#{name}"].inspect}") + if renderer + %{s << "#{name}: \#{#{naming.module_name}::Bytes.#{renderer}(defi["#{name}"].bytes)}"} + elsif be_class + render_packed_struct(name, be_class) + else + %(s << "#{name}: \#{defi["#{name}"].inspect}") + end when 'array_dynamic', 'array_static' element = fc[:element_field_class] unless %w[integer_signed integer_unsigned].include?(element[:type]) diff --git a/utils/gen_babeltrace_model_helper.rb b/utils/gen_babeltrace_model_helper.rb index 50620d31b..9c32bd25e 100644 --- a/utils/gen_babeltrace_model_helper.rb +++ b/utils/gen_babeltrace_model_helper.rb @@ -48,7 +48,7 @@ def get_extra_fields_types_name(event) event['fields'].collect do |field| lttng = LTTng::TracepointField.new(*field) name = lttng.name.to_s - type = event['args'].find { |_t, n| n == name || n == name.gsub(/_vals?\z/, '') }[0] + type = event['args'].find { |_t, n| n == name || n == parameter_name(name) }[0] field_types_name(lttng.macro.to_s, type, name, lttng) end.flatten(1) end diff --git a/utils/gen_library_base.rb b/utils/gen_library_base.rb index fd3e4c8b9..e05e69ba2 100644 --- a/utils/gen_library_base.rb +++ b/utils/gen_library_base.rb @@ -143,7 +143,7 @@ def typedef_kind(api, t) # The walk and the classification are the same for all six backends; only the # printing differs, so each kind is a keyword argument defaulting to the shared # printer. A backend passes one only where its API really diverges -- cuda -# prepends a UUID module to matching structs, itt defers its callbacks -- and +# renders a struct's declared byte arrays, itt defers its callbacks -- and # passes `nil` for a kind its bindings do not carry, which is how omp emits # enums alone. # @@ -287,45 +287,80 @@ def print_enum_with_namespace(naming, name, enum, filter_members: ->(_m) { true EOF end -# Shared by cuda/hip/mpi. ze inlines its own -- :data/:id fields, and a UUID -# printed back to front. -def print_handle_uuid_modules - puts <<'EOF' - module Handle - def to_s - s = '{ reserved: "' - s << self[:reserved].to_a.collect { |v| "\\x%02x" % ((v + 256)%256) }.join - s << '" }' +BYTE_TYPES = %w[:char :uchar :int8 :uint8].freeze + +def byte_array_members(naming, struct_name) + typedef = naming.api.types.find { |t| t.name == struct_name } + raise "meta_parameters_struct names no such struct: #{struct_name}" unless typedef + + naming.api.struct(typedef.type).to_ffi(naming).filter_map do |member, ffi_type| + member.delete_prefix(':') if ffi_type.is_a?(Array) && BYTE_TYPES.include?(ffi_type[0].to_s) + end +end + +# Declaring one member replaces the whole `to_s`, so a byte array left +# undeclared is interpolated raw: ze_kernel_uuid_t with only `kid` prints mid +# as 16 control characters. Hence the second check. +def check_meta_parameters_struct(naming, meta_parameters_struct) + unknown = meta_parameters_struct.values.flat_map(&:values).uniq - BYTES_BODIES.keys + raise "unknown renderer: #{unknown.join(', ')}" unless unknown.empty? + + meta_parameters_struct.each do |struct_name, members| + bytes = byte_array_members(naming, struct_name) + unrenderable = members.keys - bytes + unless unrenderable.empty? + raise "#{struct_name} has no byte-array member #{unrenderable.join(', ')} " \ + "(has #{bytes.join(', ')})" end + + undeclared = bytes - members.keys + raise "#{struct_name} declares no renderer for #{undeclared.join(', ')}" unless undeclared.empty? end +end + +# See backends/README.md for what each prints and why it must be declared +# rather than guessed from the C type. +DASHED_HEX = <<~EOF + hex = ordered.collect { |v| format('%02x', v % 256) } + cuts = [0, *[4, 6, 8, 10].select { |c| c < hex.length }, hex.length] + cuts.each_cons(2).collect { |a, b| hex[a...b].join }.join('-') +EOF + +BYTES_BODIES = { + 'blob' => "bytes.pack('C*').b.inspect\n", + 'uuid' => "ordered = bytes\n#{DASHED_HEX}", + 'uuid_reversed' => "ordered = bytes.reverse\n#{DASHED_HEX}", +}.freeze + +def print_bytes_module(naming, meta_parameters) + sections = meta_parameters.values_at(:meta_parameters_struct, :meta_parameters_function) + check_meta_parameters_struct(naming, sections.first) + wanted = sections.flat_map { |rows| rows.values.flat_map(&:values) }.uniq.sort + return if wanted.empty? + + puts ' module Bytes' + puts wanted.collect { |name| + body = BYTES_BODIES.fetch(name).lines.collect { |l| " #{l}" }.join + " def self.#{name}(bytes)\n#{body} end\n" + }.join("\n") + puts ' end' + puts +end + +def struct_to_s_definition(naming, struct, members) + return nil unless members + + rendered = struct.to_ffi(naming).collect do |member, _type| + key = member.delete_prefix(':') + renderer = members[key] + value = renderer ? "\#{Bytes.#{renderer}(self[#{member}].to_a)}" : "\#{self[#{member}]}" + "#{key}: #{value}" + end + < 1 }.keys + raise "#{path}: #{twice.join(', ')} declared twice" unless twice.empty? + + list.to_h(&:reverse) + end +end + # Raise unless every function the spec names is one of `commands`. A spec is # written by hand against an API that keeps moving, so a key matching nothing # is a typo or a function that has since been dropped -- either way its rows