From 7e78980b83cc84114c7347d6e20e69c4dc49fac2 Mon Sep 17 00:00:00 2001 From: "guangshui.li" Date: Thu, 24 Mar 2022 16:25:16 +0800 Subject: [PATCH 1/2] iosdiag: Add hangdetect tool Signed-off-by: guangshui.li --- source/lib/internal/kernel_module/Makefile | 10 + .../include/3.10.0-1062.1.1.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1062.1.1.el7.x86_64/virtio_blk.h | 175 +++++++++ .../include/3.10.0-1062.1.2.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1062.1.2.el7.x86_64/virtio_blk.h | 175 +++++++++ .../3.10.0-1062.12.1.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1062.12.1.el7.x86_64/virtio_blk.h | 175 +++++++++ .../3.10.0-1062.18.1.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1062.18.1.el7.x86_64/virtio_blk.h | 175 +++++++++ .../include/3.10.0-1062.4.1.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1062.4.1.el7.x86_64/virtio_blk.h | 175 +++++++++ .../include/3.10.0-1062.4.2.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1062.4.2.el7.x86_64/virtio_blk.h | 175 +++++++++ .../include/3.10.0-1062.4.3.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1062.4.3.el7.x86_64/virtio_blk.h | 175 +++++++++ .../include/3.10.0-1062.7.1.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1062.7.1.el7.x86_64/virtio_blk.h | 175 +++++++++ .../include/3.10.0-1062.9.1.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1062.9.1.el7.x86_64/virtio_blk.h | 175 +++++++++ .../include/3.10.0-1062.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1062.el7.x86_64/virtio_blk.h | 175 +++++++++ .../3.10.0-1127.10.1.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1127.10.1.el7.x86_64/virtio_blk.h | 175 +++++++++ .../3.10.0-1127.13.1.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1127.13.1.el7.x86_64/virtio_blk.h | 175 +++++++++ .../3.10.0-1127.18.2.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1127.18.2.el7.x86_64/virtio_blk.h | 175 +++++++++ .../3.10.0-1127.19.1.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1127.19.1.el7.x86_64/virtio_blk.h | 175 +++++++++ .../include/3.10.0-1127.8.2.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1127.8.2.el7.x86_64/virtio_blk.h | 175 +++++++++ .../include/3.10.0-1127.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1127.el7.x86_64/virtio_blk.h | 175 +++++++++ .../3.10.0-1160.11.1.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1160.11.1.el7.x86_64/virtio_blk.h | 175 +++++++++ .../3.10.0-1160.15.2.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1160.15.2.el7.x86_64/virtio_blk.h | 175 +++++++++ .../include/3.10.0-1160.2.1.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1160.2.1.el7.x86_64/virtio_blk.h | 175 +++++++++ .../include/3.10.0-1160.2.2.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1160.2.2.el7.x86_64/virtio_blk.h | 175 +++++++++ .../include/3.10.0-1160.6.1.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1160.6.1.el7.x86_64/virtio_blk.h | 175 +++++++++ .../include/3.10.0-1160.el7.x86_64/nvme.h | 61 ++++ .../3.10.0-1160.el7.x86_64/virtio_blk.h | 175 +++++++++ .../include/3.10.0-862.14.4.el7.x86_64/nvme.h | 60 +++ .../3.10.0-862.14.4.el7.x86_64/virtio_blk.h | 175 +++++++++ .../include/3.10.0-957.21.3.el7.x86_64/nvme.h | 60 +++ .../3.10.0-957.21.3.el7.x86_64/virtio_blk.h | 175 +++++++++ .../include/4.19.24-9.al7.x86_64/nvme.h | 60 +++ .../include/4.19.24-9.al7.x86_64/virtio_blk.h | 221 ++++++++++++ .../include/4.19.81-17.1.al7.x86_64/nvme.h | 60 +++ .../4.19.81-17.1.al7.x86_64/virtio_blk.h | 243 +++++++++++++ .../include/4.19.81-17.2.al7.x86_64/nvme.h | 60 +++ .../4.19.81-17.2.al7.x86_64/virtio_blk.h | 243 +++++++++++++ .../include/4.19.91-18.al7.x86_64/nvme.h | 60 +++ .../4.19.91-18.al7.x86_64/virtio_blk.h | 249 +++++++++++++ .../include/4.19.91-19.1.al7.x86_64/nvme.h | 60 +++ .../4.19.91-19.1.al7.x86_64/virtio_blk.h | 249 +++++++++++++ .../include/4.19.91-19.2.al7.x86_64/nvme.h | 60 +++ .../4.19.91-19.2.al7.x86_64/virtio_blk.h | 249 +++++++++++++ .../include/4.19.91-21.2.al7.x86_64/nvme.h | 62 ++++ .../4.19.91-21.2.al7.x86_64/virtio_blk.h | 292 +++++++++++++++ .../include/4.19.91-21.al7.x86_64/nvme.h | 62 ++++ .../4.19.91-21.al7.x86_64/virtio_blk.h | 292 +++++++++++++++ .../include/4.19.91-22.1.al7.x86_64/nvme.h | 62 ++++ .../4.19.91-22.1.al7.x86_64/virtio_blk.h | 292 +++++++++++++++ .../include/4.19.91-22.2.al7.x86_64/nvme.h | 62 ++++ .../4.19.91-22.2.al7.x86_64/virtio_blk.h | 292 +++++++++++++++ .../include/4.19.91-23.4.an8.x86_64/nvme.h | 62 ++++ .../4.19.91-23.4.an8.x86_64/virtio_blk.h | 292 +++++++++++++++ .../include/4.19.91-23.al7.x86_64/nvme.h | 62 ++++ .../4.19.91-23.al7.x86_64/virtio_blk.h | 292 +++++++++++++++ .../include/4.19.91-24.1.al7.x86_64/nvme.h | 62 ++++ .../4.19.91-24.1.al7.x86_64/virtio_blk.h | 292 +++++++++++++++ .../include/4.19.91-24.8.an8.x86_64/nvme.h | 62 ++++ .../4.19.91-24.8.an8.x86_64/virtio_blk.h | 292 +++++++++++++++ .../include/4.19.91-24.al7.x86_64/nvme.h | 62 ++++ .../4.19.91-24.al7.x86_64/virtio_blk.h | 292 +++++++++++++++ .../include/4.19.91-25.an8.x86_64/nvme.h | 62 ++++ .../4.19.91-25.an8.x86_64/virtio_blk.h | 292 +++++++++++++++ .../kernel_module/modules/iosdiag/iosdiag.c | 341 ++++++++++++++++++ .../kernel_module/modules/iosdiag/iosdiag.h | 101 ++++++ .../kernel_module/modules/iosdiag/nvme.c | 65 ++++ .../kernel_module/modules/iosdiag/rq_hang.c | 322 +++++++++++++++++ .../kernel_module/modules/iosdiag/scsi.c | 28 ++ .../modules/iosdiag/virtio_blk.c | 132 +++++++ .../lib/internal/kernel_module/sysak_mods.c | 9 + .../data_analysis/iosdiag_data_analysis.py | 187 +++++++++- source/tools/detect/iosdiag/entry/iosdiag.sh | 88 ++++- .../tools/detect/iosdiag/hangdetect/Makefile | 11 + .../detect/iosdiag/hangdetect/base_info.c | 159 ++++++++ .../detect/iosdiag/hangdetect/base_info.h | 49 +++ .../tools/detect/iosdiag/hangdetect/iosdiag.h | 82 +++++ .../detect/iosdiag/hangdetect/json_format.c | 284 +++++++++++++++ .../detect/iosdiag/hangdetect/json_format.h | 9 + source/tools/detect/iosdiag/hangdetect/main.c | 339 +++++++++++++++++ source/tools/detect/iosdiag/latency/main.c | 8 +- 98 files changed, 13209 insertions(+), 31 deletions(-) create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.1.1.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.1.1.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.1.2.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.1.2.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.12.1.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.12.1.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.18.1.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.18.1.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.1.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.1.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.2.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.2.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.3.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.3.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.7.1.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.7.1.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.9.1.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.9.1.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.10.1.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.10.1.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.13.1.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.13.1.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.18.2.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.18.2.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.19.1.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.19.1.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.8.2.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.8.2.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.11.1.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.11.1.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.15.2.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.15.2.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.2.1.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.2.1.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.2.2.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.2.2.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.6.1.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.6.1.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-862.14.4.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-862.14.4.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-957.21.3.el7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-957.21.3.el7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.24-9.al7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.24-9.al7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.81-17.1.al7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.81-17.1.al7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.81-17.2.al7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.81-17.2.al7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-18.al7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-18.al7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-19.1.al7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-19.1.al7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-19.2.al7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-19.2.al7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-21.2.al7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-21.2.al7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-21.al7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-21.al7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-22.1.al7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-22.1.al7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-22.2.al7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-22.2.al7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-23.4.an8.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-23.4.an8.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-23.al7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-23.al7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.1.al7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.1.al7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.8.an8.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.8.an8.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.al7.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.al7.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-25.an8.x86_64/nvme.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-25.an8.x86_64/virtio_blk.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/iosdiag.c create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/iosdiag.h create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/nvme.c create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/rq_hang.c create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/scsi.c create mode 100644 source/lib/internal/kernel_module/modules/iosdiag/virtio_blk.c create mode 100644 source/tools/detect/iosdiag/hangdetect/Makefile create mode 100644 source/tools/detect/iosdiag/hangdetect/base_info.c create mode 100644 source/tools/detect/iosdiag/hangdetect/base_info.h create mode 100644 source/tools/detect/iosdiag/hangdetect/iosdiag.h create mode 100644 source/tools/detect/iosdiag/hangdetect/json_format.c create mode 100644 source/tools/detect/iosdiag/hangdetect/json_format.h create mode 100644 source/tools/detect/iosdiag/hangdetect/main.c diff --git a/source/lib/internal/kernel_module/Makefile b/source/lib/internal/kernel_module/Makefile index a00c4984..f9200491 100644 --- a/source/lib/internal/kernel_module/Makefile +++ b/source/lib/internal/kernel_module/Makefile @@ -36,6 +36,11 @@ endif ifneq ($(findstring loadtask,$(TARGET_LIST)),) sysak-objs += modules/loadtask/loadtask.o endif +ifneq ($(findstring iosdiag,$(TARGET_LIST)),) +ifneq ($(wildcard $(MODULE_SRC)/modules/iosdiag/include/$(KERNEL_VERSION)),) +sysak-objs += modules/iosdiag/iosdiag.o modules/iosdiag/rq_hang.o modules/iosdiag/virtio_blk.o modules/iosdiag/nvme.o modules/iosdiag/scsi.o +endif +endif obj-m += sysak.o @@ -43,6 +48,11 @@ obj-m += sysak.o EXTRA_CFLAGS := -I$(MODULE_SRC) EXTRA_CFLAGS += -I$(MODULE_SRC)/include EXTRA_CFLAGS += -I$(SRC)/lib/uapi/include +ifneq ($(findstring iosdiag,$(TARGET_LIST)),) +ifneq ($(wildcard $(MODULE_SRC)/modules/iosdiag/include/$(KERNEL_VERSION)),) +EXTRA_CFLAGS += -I$(MODULE_SRC)/modules/iosdiag -I$(MODULE_SRC)/modules/iosdiag/include/$(KERNEL_VERSION) +endif +endif sysak_mod: make -C $(KERNEL_BUILD_PATH) M=$(MODULE_SRC) diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.1.1.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.1.1.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.1.1.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.1.1.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.1.1.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.1.1.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.1.2.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.1.2.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.1.2.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.1.2.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.1.2.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.1.2.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.12.1.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.12.1.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.12.1.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.12.1.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.12.1.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.12.1.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.18.1.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.18.1.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.18.1.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.18.1.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.18.1.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.18.1.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.1.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.1.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.1.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.1.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.1.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.1.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.2.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.2.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.2.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.2.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.2.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.2.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.3.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.3.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.3.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.3.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.3.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.4.3.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.7.1.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.7.1.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.7.1.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.7.1.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.7.1.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.7.1.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.9.1.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.9.1.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.9.1.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.9.1.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.9.1.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.9.1.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1062.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.10.1.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.10.1.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.10.1.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.10.1.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.10.1.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.10.1.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.13.1.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.13.1.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.13.1.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.13.1.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.13.1.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.13.1.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.18.2.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.18.2.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.18.2.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.18.2.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.18.2.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.18.2.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.19.1.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.19.1.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.19.1.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.19.1.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.19.1.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.19.1.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.8.2.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.8.2.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.8.2.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.8.2.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.8.2.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.8.2.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1127.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.11.1.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.11.1.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.11.1.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.11.1.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.11.1.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.11.1.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.15.2.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.15.2.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.15.2.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.15.2.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.15.2.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.15.2.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.2.1.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.2.1.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.2.1.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.2.1.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.2.1.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.2.1.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.2.2.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.2.2.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.2.2.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.2.2.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.2.2.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.2.2.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.6.1.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.6.1.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.6.1.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.6.1.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.6.1.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.6.1.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.el7.x86_64/nvme.h new file mode 100644 index 00000000..628ed56d --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.el7.x86_64/nvme.h @@ -0,0 +1,61 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-1160.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-862.14.4.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-862.14.4.el7.x86_64/nvme.h new file mode 100644 index 00000000..cf9e37b2 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-862.14.4.el7.x86_64/nvme.h @@ -0,0 +1,60 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 qid; + u8 cq_phase; + u8 cqe_seen; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-862.14.4.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-862.14.4.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-862.14.4.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-957.21.3.el7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-957.21.3.el7.x86_64/nvme.h new file mode 100644 index 00000000..cf9e37b2 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-957.21.3.el7.x86_64/nvme.h @@ -0,0 +1,60 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + char irqname[24]; /* nvme4294967295-65535\0 */ + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 qid; + u8 cq_phase; + u8 cqe_seen; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-957.21.3.el7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-957.21.3.el7.x86_64/virtio_blk.h new file mode 100644 index 00000000..faa22a32 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/3.10.0-957.21.3.el7.x86_64/virtio_blk.h @@ -0,0 +1,175 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { + struct request *req; + struct virtio_blk_outhdr out_hdr; + struct virtio_scsi_inhdr in_hdr; + u8 status; + struct scatterlist sg[]; +}; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + RH_KABI_DEPRECATE(unsigned int, ipi_redirect) + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? ((struct virtblk_req *)data)->req : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->timeout) + return jiffies_to_usecs(rq->deadline - rq->timeout) * 1000; + return 0; +} + +static inline u64 get_check_hang_time_ns(void) +{ + return sched_clock(); +} + +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + blk_mq_tagset_busy_iter(q->tag_set, fn, data); + return 0; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.24-9.al7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.24-9.al7.x86_64/nvme.h new file mode 100644 index 00000000..500b3bc2 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.24-9.al7.x86_64/nvme.h @@ -0,0 +1,60 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.24-9.al7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.24-9.al7.x86_64/virtio_blk.h new file mode 100644 index 00000000..f76c300e --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.24-9.al7.x86_64/virtio_blk.h @@ -0,0 +1,221 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* Last written value to avail->flags */ + u16 avail_flags_shadow; + + /* Last written value to avail->idx in guest byte order */ + u16 avail_idx_shadow; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { +#ifdef CONFIG_VIRTIO_BLK_SCSI + struct scsi_request sreq; /* for SCSI passthrough, must be first */ + u8 sense[SCSI_SENSE_BUFFERSIZE]; + struct virtio_scsi_inhdr in_hdr; +#endif + struct virtio_blk_outhdr out_hdr; + u8 status; + struct scatterlist sg[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +struct blk_flush_queue { + unsigned int flush_queue_delayed:1; + unsigned int flush_pending_idx:1; + unsigned int flush_running_idx:1; + unsigned long flush_pending_since; + struct list_head flush_queue[2]; + struct list_head flush_data_in_flight; + struct request *flush_rq; + + /* + * flush_rq shares tag with this rq, both can't be active + * at the same time + */ + struct request *orig_rq; + spinlock_t mq_flush_lock; +}; + +//only kernel ali4000 >= 007 +static inline int enable_detect_flush_rq(void) +{ + return 0; +} + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? blk_mq_rq_from_pdu(data) : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return rq ? rq->internal_tag : -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->__deadline > rq->timeout) + return jiffies_to_usecs(rq->__deadline - rq->timeout) * 1000; + return 0; +} + +/* + * (ALIKERNEL && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 151)) || + * LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0) + */ +static inline u64 get_check_hang_time_ns(void) +{ + return ktime_get_ns(); +} + +//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +extern fn_queue_tag_busy_iter sym_blk_mq_queue_tag_busy_iter; +//#endif +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static blk_mq_rq_iter fn_blk_mq_check_hang = NULL; +static void blk_mq_check_rq_hang(struct blk_mq_hw_ctx *hctx, + struct request *rq, void *priv, bool reserved) +{ + if (fn_blk_mq_check_hang) + fn_blk_mq_check_hang(rq, priv, reserved); +} + +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + fn_blk_mq_check_hang = fn; + sym_blk_mq_queue_tag_busy_iter(q, blk_mq_check_rq_hang, data); + return 0; +} +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.81-17.1.al7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.81-17.1.al7.x86_64/nvme.h new file mode 100644 index 00000000..500b3bc2 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.81-17.1.al7.x86_64/nvme.h @@ -0,0 +1,60 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.81-17.1.al7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.81-17.1.al7.x86_64/virtio_blk.h new file mode 100644 index 00000000..8aed3694 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.81-17.1.al7.x86_64/virtio_blk.h @@ -0,0 +1,243 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* Last written value to avail->flags */ + u16 avail_flags_shadow; + + /* Last written value to avail->idx in guest byte order */ + u16 avail_idx_shadow; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { +#ifdef CONFIG_VIRTIO_BLK_SCSI + struct scsi_request sreq; /* for SCSI passthrough, must be first */ + u8 sense[SCSI_SENSE_BUFFERSIZE]; + struct virtio_scsi_inhdr in_hdr; +#endif + struct virtio_blk_outhdr out_hdr; + u8 status; + struct scatterlist sg[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +struct blk_flush_queue { + unsigned int flush_queue_delayed:1; + unsigned int flush_pending_idx:1; + unsigned int flush_running_idx:1; + blk_status_t rq_status; + unsigned long flush_pending_since; + struct list_head flush_queue[2]; + struct list_head flush_data_in_flight; + struct request *flush_rq; + + /* + * flush_rq shares tag with this rq, both can't be active + * at the same time + */ + struct request *orig_rq; + spinlock_t mq_flush_lock; +}; + +//only kernel ali4000 >= 007 +static inline int enable_detect_flush_rq(void) +{ + return 1; +} + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? blk_mq_rq_from_pdu(data) : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return rq ? rq->internal_tag : -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->__deadline > rq->timeout) + return jiffies_to_usecs(rq->__deadline - rq->timeout) * 1000; + return 0; +} + +/* + * (ALIKERNEL && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 151)) || + * LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0) + */ +static inline u64 get_check_hang_time_ns(void) +{ + return ktime_get_ns(); +} + +//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +extern fn_queue_tag_busy_iter sym_blk_mq_queue_tag_busy_iter; +//#endif +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static blk_mq_rq_iter fn_blk_mq_check_hang = NULL; +static void blk_mq_check_rq_hang(struct blk_mq_hw_ctx *hctx, + struct request *rq, void *priv, bool reserved) +{ + if (fn_blk_mq_check_hang) + fn_blk_mq_check_hang(rq, priv, reserved); +} + +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + fn_blk_mq_check_hang = fn; +/* +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + if (!percpu_ref_tryget(&q->q_usage_counter)) + return -EBUSY; +#endif +*/ + sym_blk_mq_queue_tag_busy_iter(q, blk_mq_check_rq_hang, data); +/* +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + blk_queue_exit(q); +#endif +#else + struct blk_mq_hw_ctx *hctx; + int i; + + queue_for_each_hw_ctx(q, hctx, i) { + if (!(hctx->nr_ctx && hctx->tags)) + continue; + blk_mq_tag_busy_iter(hctx, blk_mq_check_rq_hang, data); + } +#endif +*/ + return 0; +} +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.81-17.2.al7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.81-17.2.al7.x86_64/nvme.h new file mode 100644 index 00000000..500b3bc2 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.81-17.2.al7.x86_64/nvme.h @@ -0,0 +1,60 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.81-17.2.al7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.81-17.2.al7.x86_64/virtio_blk.h new file mode 100644 index 00000000..8aed3694 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.81-17.2.al7.x86_64/virtio_blk.h @@ -0,0 +1,243 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* Last written value to avail->flags */ + u16 avail_flags_shadow; + + /* Last written value to avail->idx in guest byte order */ + u16 avail_idx_shadow; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { +#ifdef CONFIG_VIRTIO_BLK_SCSI + struct scsi_request sreq; /* for SCSI passthrough, must be first */ + u8 sense[SCSI_SENSE_BUFFERSIZE]; + struct virtio_scsi_inhdr in_hdr; +#endif + struct virtio_blk_outhdr out_hdr; + u8 status; + struct scatterlist sg[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +struct blk_flush_queue { + unsigned int flush_queue_delayed:1; + unsigned int flush_pending_idx:1; + unsigned int flush_running_idx:1; + blk_status_t rq_status; + unsigned long flush_pending_since; + struct list_head flush_queue[2]; + struct list_head flush_data_in_flight; + struct request *flush_rq; + + /* + * flush_rq shares tag with this rq, both can't be active + * at the same time + */ + struct request *orig_rq; + spinlock_t mq_flush_lock; +}; + +//only kernel ali4000 >= 007 +static inline int enable_detect_flush_rq(void) +{ + return 1; +} + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? blk_mq_rq_from_pdu(data) : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return rq ? rq->internal_tag : -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->__deadline > rq->timeout) + return jiffies_to_usecs(rq->__deadline - rq->timeout) * 1000; + return 0; +} + +/* + * (ALIKERNEL && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 151)) || + * LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0) + */ +static inline u64 get_check_hang_time_ns(void) +{ + return ktime_get_ns(); +} + +//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +extern fn_queue_tag_busy_iter sym_blk_mq_queue_tag_busy_iter; +//#endif +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static blk_mq_rq_iter fn_blk_mq_check_hang = NULL; +static void blk_mq_check_rq_hang(struct blk_mq_hw_ctx *hctx, + struct request *rq, void *priv, bool reserved) +{ + if (fn_blk_mq_check_hang) + fn_blk_mq_check_hang(rq, priv, reserved); +} + +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + fn_blk_mq_check_hang = fn; +/* +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + if (!percpu_ref_tryget(&q->q_usage_counter)) + return -EBUSY; +#endif +*/ + sym_blk_mq_queue_tag_busy_iter(q, blk_mq_check_rq_hang, data); +/* +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + blk_queue_exit(q); +#endif +#else + struct blk_mq_hw_ctx *hctx; + int i; + + queue_for_each_hw_ctx(q, hctx, i) { + if (!(hctx->nr_ctx && hctx->tags)) + continue; + blk_mq_tag_busy_iter(hctx, blk_mq_check_rq_hang, data); + } +#endif +*/ + return 0; +} +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-18.al7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-18.al7.x86_64/nvme.h new file mode 100644 index 00000000..500b3bc2 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-18.al7.x86_64/nvme.h @@ -0,0 +1,60 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-18.al7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-18.al7.x86_64/virtio_blk.h new file mode 100644 index 00000000..55f4db57 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-18.al7.x86_64/virtio_blk.h @@ -0,0 +1,249 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* Last written value to avail->flags */ + u16 avail_flags_shadow; + + /* Last written value to avail->idx in guest byte order */ + u16 avail_idx_shadow; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { +#ifdef CONFIG_VIRTIO_BLK_SCSI + struct scsi_request sreq; /* for SCSI passthrough, must be first */ + u8 sense[SCSI_SENSE_BUFFERSIZE]; + struct virtio_scsi_inhdr in_hdr; +#endif + struct virtio_blk_outhdr out_hdr; + u8 status; + struct scatterlist sg[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctxs { + struct kobject kobj; + struct blk_mq_ctx __percpu *queue_ctx; +}; + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct blk_mq_ctxs *ctxs; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +struct blk_flush_queue { + unsigned int flush_queue_delayed:1; + unsigned int flush_pending_idx:1; + unsigned int flush_running_idx:1; + blk_status_t rq_status; + unsigned long flush_pending_since; + struct list_head flush_queue[2]; + struct list_head flush_data_in_flight; + struct request *flush_rq; + + /* + * flush_rq shares tag with this rq, both can't be active + * at the same time + */ + struct request *orig_rq; + spinlock_t mq_flush_lock; +}; + +//only kernel ali4000 >= 007 +static inline int enable_detect_flush_rq(void) +{ + return 1; +} + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? blk_mq_rq_from_pdu(data) : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return rq ? rq->internal_tag : -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->__deadline > rq->timeout) + return jiffies_to_usecs(rq->__deadline - rq->timeout) * 1000; + return 0; +} + +/* + * (ALIKERNEL && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 151)) || + * LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0) + */ +static inline u64 get_check_hang_time_ns(void) +{ + return ktime_get_ns(); +} + +//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +extern fn_queue_tag_busy_iter sym_blk_mq_queue_tag_busy_iter; +//#endif +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static blk_mq_rq_iter fn_blk_mq_check_hang = NULL; +static void blk_mq_check_rq_hang(struct blk_mq_hw_ctx *hctx, + struct request *rq, void *priv, bool reserved) +{ + if (fn_blk_mq_check_hang) + fn_blk_mq_check_hang(rq, priv, reserved); +} + +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + fn_blk_mq_check_hang = fn; +/* +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + if (!percpu_ref_tryget(&q->q_usage_counter)) + return -EBUSY; +#endif +*/ + sym_blk_mq_queue_tag_busy_iter(q, blk_mq_check_rq_hang, data); +/* +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + blk_queue_exit(q); +#endif +#else + struct blk_mq_hw_ctx *hctx; + int i; + + queue_for_each_hw_ctx(q, hctx, i) { + if (!(hctx->nr_ctx && hctx->tags)) + continue; + blk_mq_tag_busy_iter(hctx, blk_mq_check_rq_hang, data); + } +#endif +*/ + return 0; +} +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-19.1.al7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-19.1.al7.x86_64/nvme.h new file mode 100644 index 00000000..500b3bc2 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-19.1.al7.x86_64/nvme.h @@ -0,0 +1,60 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-19.1.al7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-19.1.al7.x86_64/virtio_blk.h new file mode 100644 index 00000000..55f4db57 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-19.1.al7.x86_64/virtio_blk.h @@ -0,0 +1,249 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* Last written value to avail->flags */ + u16 avail_flags_shadow; + + /* Last written value to avail->idx in guest byte order */ + u16 avail_idx_shadow; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { +#ifdef CONFIG_VIRTIO_BLK_SCSI + struct scsi_request sreq; /* for SCSI passthrough, must be first */ + u8 sense[SCSI_SENSE_BUFFERSIZE]; + struct virtio_scsi_inhdr in_hdr; +#endif + struct virtio_blk_outhdr out_hdr; + u8 status; + struct scatterlist sg[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctxs { + struct kobject kobj; + struct blk_mq_ctx __percpu *queue_ctx; +}; + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct blk_mq_ctxs *ctxs; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +struct blk_flush_queue { + unsigned int flush_queue_delayed:1; + unsigned int flush_pending_idx:1; + unsigned int flush_running_idx:1; + blk_status_t rq_status; + unsigned long flush_pending_since; + struct list_head flush_queue[2]; + struct list_head flush_data_in_flight; + struct request *flush_rq; + + /* + * flush_rq shares tag with this rq, both can't be active + * at the same time + */ + struct request *orig_rq; + spinlock_t mq_flush_lock; +}; + +//only kernel ali4000 >= 007 +static inline int enable_detect_flush_rq(void) +{ + return 1; +} + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? blk_mq_rq_from_pdu(data) : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return rq ? rq->internal_tag : -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->__deadline > rq->timeout) + return jiffies_to_usecs(rq->__deadline - rq->timeout) * 1000; + return 0; +} + +/* + * (ALIKERNEL && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 151)) || + * LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0) + */ +static inline u64 get_check_hang_time_ns(void) +{ + return ktime_get_ns(); +} + +//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +extern fn_queue_tag_busy_iter sym_blk_mq_queue_tag_busy_iter; +//#endif +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static blk_mq_rq_iter fn_blk_mq_check_hang = NULL; +static void blk_mq_check_rq_hang(struct blk_mq_hw_ctx *hctx, + struct request *rq, void *priv, bool reserved) +{ + if (fn_blk_mq_check_hang) + fn_blk_mq_check_hang(rq, priv, reserved); +} + +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + fn_blk_mq_check_hang = fn; +/* +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + if (!percpu_ref_tryget(&q->q_usage_counter)) + return -EBUSY; +#endif +*/ + sym_blk_mq_queue_tag_busy_iter(q, blk_mq_check_rq_hang, data); +/* +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + blk_queue_exit(q); +#endif +#else + struct blk_mq_hw_ctx *hctx; + int i; + + queue_for_each_hw_ctx(q, hctx, i) { + if (!(hctx->nr_ctx && hctx->tags)) + continue; + blk_mq_tag_busy_iter(hctx, blk_mq_check_rq_hang, data); + } +#endif +*/ + return 0; +} +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-19.2.al7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-19.2.al7.x86_64/nvme.h new file mode 100644 index 00000000..500b3bc2 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-19.2.al7.x86_64/nvme.h @@ -0,0 +1,60 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-19.2.al7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-19.2.al7.x86_64/virtio_blk.h new file mode 100644 index 00000000..55f4db57 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-19.2.al7.x86_64/virtio_blk.h @@ -0,0 +1,249 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* Last written value to avail->flags */ + u16 avail_flags_shadow; + + /* Last written value to avail->idx in guest byte order */ + u16 avail_idx_shadow; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; + +struct virtio_blk { + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { +#ifdef CONFIG_VIRTIO_BLK_SCSI + struct scsi_request sreq; /* for SCSI passthrough, must be first */ + u8 sense[SCSI_SENSE_BUFFERSIZE]; + struct virtio_scsi_inhdr in_hdr; +#endif + struct virtio_blk_outhdr out_hdr; + u8 status; + struct scatterlist sg[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctxs { + struct kobject kobj; + struct blk_mq_ctx __percpu *queue_ctx; +}; + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned int index_hw; + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct blk_mq_ctxs *ctxs; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +struct blk_flush_queue { + unsigned int flush_queue_delayed:1; + unsigned int flush_pending_idx:1; + unsigned int flush_running_idx:1; + blk_status_t rq_status; + unsigned long flush_pending_since; + struct list_head flush_queue[2]; + struct list_head flush_data_in_flight; + struct request *flush_rq; + + /* + * flush_rq shares tag with this rq, both can't be active + * at the same time + */ + struct request *orig_rq; + spinlock_t mq_flush_lock; +}; + +//only kernel ali4000 >= 007 +static inline int enable_detect_flush_rq(void) +{ + return 1; +} + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + int cpu) +{ + return q->queue_hw_ctx[q->mq_map[cpu]]; +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? blk_mq_rq_from_pdu(data) : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return rq ? rq->internal_tag : -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->__deadline > rq->timeout) + return jiffies_to_usecs(rq->__deadline - rq->timeout) * 1000; + return 0; +} + +/* + * (ALIKERNEL && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 151)) || + * LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0) + */ +static inline u64 get_check_hang_time_ns(void) +{ + return ktime_get_ns(); +} + +//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +extern fn_queue_tag_busy_iter sym_blk_mq_queue_tag_busy_iter; +//#endif +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static blk_mq_rq_iter fn_blk_mq_check_hang = NULL; +static void blk_mq_check_rq_hang(struct blk_mq_hw_ctx *hctx, + struct request *rq, void *priv, bool reserved) +{ + if (fn_blk_mq_check_hang) + fn_blk_mq_check_hang(rq, priv, reserved); +} + +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + fn_blk_mq_check_hang = fn; +/* +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + if (!percpu_ref_tryget(&q->q_usage_counter)) + return -EBUSY; +#endif +*/ + sym_blk_mq_queue_tag_busy_iter(q, blk_mq_check_rq_hang, data); +/* +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + blk_queue_exit(q); +#endif +#else + struct blk_mq_hw_ctx *hctx; + int i; + + queue_for_each_hw_ctx(q, hctx, i) { + if (!(hctx->nr_ctx && hctx->tags)) + continue; + blk_mq_tag_busy_iter(hctx, blk_mq_check_rq_hang, data); + } +#endif +*/ + return 0; +} +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-21.2.al7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-21.2.al7.x86_64/nvme.h new file mode 100644 index 00000000..35cb9aba --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-21.2.al7.x86_64/nvme.h @@ -0,0 +1,62 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 last_sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u8 polled; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-21.2.al7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-21.2.al7.x86_64/virtio_blk.h new file mode 100644 index 00000000..e7793610 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-21.2.al7.x86_64/virtio_blk.h @@ -0,0 +1,292 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* Last written value to avail->flags */ + u16 avail_flags_shadow; + + /* Last written value to avail->idx in guest byte order */ + u16 avail_idx_shadow; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; + +struct virtio_blk { + /* + * This mutex must be held by anything that may run after + * virtblk_remove() sets vblk->vdev to NULL. + * + * blk-mq, virtqueue processing, and sysfs attribute code paths are + * shut down before vblk->vdev is set to NULL and therefore do not need + * to hold this mutex. + */ + struct mutex vdev_mutex; + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* + * Tracks references from block_device_operations open/release and + * virtio_driver probe/remove so this object can be freed once no + * longer in use. + */ + refcount_t refs; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { +#ifdef CONFIG_VIRTIO_BLK_SCSI + struct scsi_request sreq; /* for SCSI passthrough, must be first */ + u8 sense[SCSI_SENSE_BUFFERSIZE]; + struct virtio_scsi_inhdr in_hdr; +#endif + struct virtio_blk_outhdr out_hdr; + u8 status; + struct scatterlist sg[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctxs { + struct kobject kobj; + struct blk_mq_ctx __percpu *queue_ctx; +}; + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned short index_hw[HCTX_MAX_TYPES]; + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct blk_mq_ctxs *ctxs; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +struct blk_flush_queue { + unsigned int flush_queue_delayed:1; + unsigned int flush_pending_idx:1; + unsigned int flush_running_idx:1; + blk_status_t rq_status; + unsigned long flush_pending_since; + struct list_head flush_queue[2]; + struct list_head flush_data_in_flight; + struct request *flush_rq; + + /* + * flush_rq shares tag with this rq, both can't be active + * at the same time + */ + struct request *orig_rq; + spinlock_t mq_flush_lock; +}; + +//only kernel ali4000 >= 007 +static inline int enable_detect_flush_rq(void) +{ + return 1; +} + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue_type(struct request_queue *q, + enum hctx_type type, + unsigned int cpu) +{ + return q->queue_hw_ctx[q->tag_set->map[type].mq_map[cpu]]; +} + +/* + * blk_mq_map_queue() - map (cmd_flags,type) to hardware queue + * @q: request queue + * @flags: request command flags + * @cpu: CPU + */ +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + unsigned int flags, + unsigned int cpu) +{ + enum hctx_type type = HCTX_TYPE_DEFAULT; + + if ((flags & REQ_HIPRI) && + q->tag_set->nr_maps > HCTX_TYPE_POLL && + q->tag_set->map[HCTX_TYPE_POLL].nr_queues && + test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) + type = HCTX_TYPE_POLL; + + else if (((flags & REQ_OP_MASK) == REQ_OP_READ) && + q->tag_set->nr_maps > HCTX_TYPE_READ && + q->tag_set->map[HCTX_TYPE_READ].nr_queues) + type = HCTX_TYPE_READ; + + return blk_mq_map_queue_type(q, type, cpu); +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->cmd_flags, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? blk_mq_rq_from_pdu(data) : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return rq ? rq->internal_tag : -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->__deadline > rq->timeout) + return jiffies_to_usecs(rq->__deadline - rq->timeout) * 1000; + return 0; +} + +/* + * (ALIKERNEL && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 151)) || + * LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0) + */ +static inline u64 get_check_hang_time_ns(void) +{ + return ktime_get_ns(); +} + +//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +extern fn_queue_tag_busy_iter sym_blk_mq_queue_tag_busy_iter; +//#endif +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static blk_mq_rq_iter fn_blk_mq_check_hang = NULL; +static void blk_mq_check_rq_hang(struct blk_mq_hw_ctx *hctx, + struct request *rq, void *priv, bool reserved) +{ + if (fn_blk_mq_check_hang) + fn_blk_mq_check_hang(rq, priv, reserved); +} + +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + fn_blk_mq_check_hang = fn; +/* +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + if (!percpu_ref_tryget(&q->q_usage_counter)) + return -EBUSY; +#endif +*/ + sym_blk_mq_queue_tag_busy_iter(q, blk_mq_check_rq_hang, data); +/* +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + blk_queue_exit(q); +#endif +#else + struct blk_mq_hw_ctx *hctx; + int i; + + queue_for_each_hw_ctx(q, hctx, i) { + if (!(hctx->nr_ctx && hctx->tags)) + continue; + blk_mq_tag_busy_iter(hctx, blk_mq_check_rq_hang, data); + } +#endif +*/ + return 0; +} +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-21.al7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-21.al7.x86_64/nvme.h new file mode 100644 index 00000000..35cb9aba --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-21.al7.x86_64/nvme.h @@ -0,0 +1,62 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 last_sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u8 polled; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-21.al7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-21.al7.x86_64/virtio_blk.h new file mode 100644 index 00000000..e7793610 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-21.al7.x86_64/virtio_blk.h @@ -0,0 +1,292 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* Last written value to avail->flags */ + u16 avail_flags_shadow; + + /* Last written value to avail->idx in guest byte order */ + u16 avail_idx_shadow; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; + +struct virtio_blk { + /* + * This mutex must be held by anything that may run after + * virtblk_remove() sets vblk->vdev to NULL. + * + * blk-mq, virtqueue processing, and sysfs attribute code paths are + * shut down before vblk->vdev is set to NULL and therefore do not need + * to hold this mutex. + */ + struct mutex vdev_mutex; + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* + * Tracks references from block_device_operations open/release and + * virtio_driver probe/remove so this object can be freed once no + * longer in use. + */ + refcount_t refs; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { +#ifdef CONFIG_VIRTIO_BLK_SCSI + struct scsi_request sreq; /* for SCSI passthrough, must be first */ + u8 sense[SCSI_SENSE_BUFFERSIZE]; + struct virtio_scsi_inhdr in_hdr; +#endif + struct virtio_blk_outhdr out_hdr; + u8 status; + struct scatterlist sg[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctxs { + struct kobject kobj; + struct blk_mq_ctx __percpu *queue_ctx; +}; + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned short index_hw[HCTX_MAX_TYPES]; + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct blk_mq_ctxs *ctxs; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +struct blk_flush_queue { + unsigned int flush_queue_delayed:1; + unsigned int flush_pending_idx:1; + unsigned int flush_running_idx:1; + blk_status_t rq_status; + unsigned long flush_pending_since; + struct list_head flush_queue[2]; + struct list_head flush_data_in_flight; + struct request *flush_rq; + + /* + * flush_rq shares tag with this rq, both can't be active + * at the same time + */ + struct request *orig_rq; + spinlock_t mq_flush_lock; +}; + +//only kernel ali4000 >= 007 +static inline int enable_detect_flush_rq(void) +{ + return 1; +} + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue_type(struct request_queue *q, + enum hctx_type type, + unsigned int cpu) +{ + return q->queue_hw_ctx[q->tag_set->map[type].mq_map[cpu]]; +} + +/* + * blk_mq_map_queue() - map (cmd_flags,type) to hardware queue + * @q: request queue + * @flags: request command flags + * @cpu: CPU + */ +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + unsigned int flags, + unsigned int cpu) +{ + enum hctx_type type = HCTX_TYPE_DEFAULT; + + if ((flags & REQ_HIPRI) && + q->tag_set->nr_maps > HCTX_TYPE_POLL && + q->tag_set->map[HCTX_TYPE_POLL].nr_queues && + test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) + type = HCTX_TYPE_POLL; + + else if (((flags & REQ_OP_MASK) == REQ_OP_READ) && + q->tag_set->nr_maps > HCTX_TYPE_READ && + q->tag_set->map[HCTX_TYPE_READ].nr_queues) + type = HCTX_TYPE_READ; + + return blk_mq_map_queue_type(q, type, cpu); +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->cmd_flags, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? blk_mq_rq_from_pdu(data) : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return rq ? rq->internal_tag : -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->__deadline > rq->timeout) + return jiffies_to_usecs(rq->__deadline - rq->timeout) * 1000; + return 0; +} + +/* + * (ALIKERNEL && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 151)) || + * LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0) + */ +static inline u64 get_check_hang_time_ns(void) +{ + return ktime_get_ns(); +} + +//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +extern fn_queue_tag_busy_iter sym_blk_mq_queue_tag_busy_iter; +//#endif +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static blk_mq_rq_iter fn_blk_mq_check_hang = NULL; +static void blk_mq_check_rq_hang(struct blk_mq_hw_ctx *hctx, + struct request *rq, void *priv, bool reserved) +{ + if (fn_blk_mq_check_hang) + fn_blk_mq_check_hang(rq, priv, reserved); +} + +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + fn_blk_mq_check_hang = fn; +/* +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + if (!percpu_ref_tryget(&q->q_usage_counter)) + return -EBUSY; +#endif +*/ + sym_blk_mq_queue_tag_busy_iter(q, blk_mq_check_rq_hang, data); +/* +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + blk_queue_exit(q); +#endif +#else + struct blk_mq_hw_ctx *hctx; + int i; + + queue_for_each_hw_ctx(q, hctx, i) { + if (!(hctx->nr_ctx && hctx->tags)) + continue; + blk_mq_tag_busy_iter(hctx, blk_mq_check_rq_hang, data); + } +#endif +*/ + return 0; +} +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-22.1.al7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-22.1.al7.x86_64/nvme.h new file mode 100644 index 00000000..35cb9aba --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-22.1.al7.x86_64/nvme.h @@ -0,0 +1,62 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 last_sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u8 polled; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-22.1.al7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-22.1.al7.x86_64/virtio_blk.h new file mode 100644 index 00000000..e7793610 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-22.1.al7.x86_64/virtio_blk.h @@ -0,0 +1,292 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* Last written value to avail->flags */ + u16 avail_flags_shadow; + + /* Last written value to avail->idx in guest byte order */ + u16 avail_idx_shadow; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; + +struct virtio_blk { + /* + * This mutex must be held by anything that may run after + * virtblk_remove() sets vblk->vdev to NULL. + * + * blk-mq, virtqueue processing, and sysfs attribute code paths are + * shut down before vblk->vdev is set to NULL and therefore do not need + * to hold this mutex. + */ + struct mutex vdev_mutex; + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* + * Tracks references from block_device_operations open/release and + * virtio_driver probe/remove so this object can be freed once no + * longer in use. + */ + refcount_t refs; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { +#ifdef CONFIG_VIRTIO_BLK_SCSI + struct scsi_request sreq; /* for SCSI passthrough, must be first */ + u8 sense[SCSI_SENSE_BUFFERSIZE]; + struct virtio_scsi_inhdr in_hdr; +#endif + struct virtio_blk_outhdr out_hdr; + u8 status; + struct scatterlist sg[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctxs { + struct kobject kobj; + struct blk_mq_ctx __percpu *queue_ctx; +}; + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned short index_hw[HCTX_MAX_TYPES]; + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct blk_mq_ctxs *ctxs; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +struct blk_flush_queue { + unsigned int flush_queue_delayed:1; + unsigned int flush_pending_idx:1; + unsigned int flush_running_idx:1; + blk_status_t rq_status; + unsigned long flush_pending_since; + struct list_head flush_queue[2]; + struct list_head flush_data_in_flight; + struct request *flush_rq; + + /* + * flush_rq shares tag with this rq, both can't be active + * at the same time + */ + struct request *orig_rq; + spinlock_t mq_flush_lock; +}; + +//only kernel ali4000 >= 007 +static inline int enable_detect_flush_rq(void) +{ + return 1; +} + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue_type(struct request_queue *q, + enum hctx_type type, + unsigned int cpu) +{ + return q->queue_hw_ctx[q->tag_set->map[type].mq_map[cpu]]; +} + +/* + * blk_mq_map_queue() - map (cmd_flags,type) to hardware queue + * @q: request queue + * @flags: request command flags + * @cpu: CPU + */ +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + unsigned int flags, + unsigned int cpu) +{ + enum hctx_type type = HCTX_TYPE_DEFAULT; + + if ((flags & REQ_HIPRI) && + q->tag_set->nr_maps > HCTX_TYPE_POLL && + q->tag_set->map[HCTX_TYPE_POLL].nr_queues && + test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) + type = HCTX_TYPE_POLL; + + else if (((flags & REQ_OP_MASK) == REQ_OP_READ) && + q->tag_set->nr_maps > HCTX_TYPE_READ && + q->tag_set->map[HCTX_TYPE_READ].nr_queues) + type = HCTX_TYPE_READ; + + return blk_mq_map_queue_type(q, type, cpu); +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->cmd_flags, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? blk_mq_rq_from_pdu(data) : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return rq ? rq->internal_tag : -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->__deadline > rq->timeout) + return jiffies_to_usecs(rq->__deadline - rq->timeout) * 1000; + return 0; +} + +/* + * (ALIKERNEL && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 151)) || + * LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0) + */ +static inline u64 get_check_hang_time_ns(void) +{ + return ktime_get_ns(); +} + +//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +extern fn_queue_tag_busy_iter sym_blk_mq_queue_tag_busy_iter; +//#endif +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static blk_mq_rq_iter fn_blk_mq_check_hang = NULL; +static void blk_mq_check_rq_hang(struct blk_mq_hw_ctx *hctx, + struct request *rq, void *priv, bool reserved) +{ + if (fn_blk_mq_check_hang) + fn_blk_mq_check_hang(rq, priv, reserved); +} + +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + fn_blk_mq_check_hang = fn; +/* +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + if (!percpu_ref_tryget(&q->q_usage_counter)) + return -EBUSY; +#endif +*/ + sym_blk_mq_queue_tag_busy_iter(q, blk_mq_check_rq_hang, data); +/* +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + blk_queue_exit(q); +#endif +#else + struct blk_mq_hw_ctx *hctx; + int i; + + queue_for_each_hw_ctx(q, hctx, i) { + if (!(hctx->nr_ctx && hctx->tags)) + continue; + blk_mq_tag_busy_iter(hctx, blk_mq_check_rq_hang, data); + } +#endif +*/ + return 0; +} +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-22.2.al7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-22.2.al7.x86_64/nvme.h new file mode 100644 index 00000000..35cb9aba --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-22.2.al7.x86_64/nvme.h @@ -0,0 +1,62 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 last_sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u8 polled; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-22.2.al7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-22.2.al7.x86_64/virtio_blk.h new file mode 100644 index 00000000..e7793610 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-22.2.al7.x86_64/virtio_blk.h @@ -0,0 +1,292 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* Last written value to avail->flags */ + u16 avail_flags_shadow; + + /* Last written value to avail->idx in guest byte order */ + u16 avail_idx_shadow; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; + +struct virtio_blk { + /* + * This mutex must be held by anything that may run after + * virtblk_remove() sets vblk->vdev to NULL. + * + * blk-mq, virtqueue processing, and sysfs attribute code paths are + * shut down before vblk->vdev is set to NULL and therefore do not need + * to hold this mutex. + */ + struct mutex vdev_mutex; + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* + * Tracks references from block_device_operations open/release and + * virtio_driver probe/remove so this object can be freed once no + * longer in use. + */ + refcount_t refs; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { +#ifdef CONFIG_VIRTIO_BLK_SCSI + struct scsi_request sreq; /* for SCSI passthrough, must be first */ + u8 sense[SCSI_SENSE_BUFFERSIZE]; + struct virtio_scsi_inhdr in_hdr; +#endif + struct virtio_blk_outhdr out_hdr; + u8 status; + struct scatterlist sg[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctxs { + struct kobject kobj; + struct blk_mq_ctx __percpu *queue_ctx; +}; + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned short index_hw[HCTX_MAX_TYPES]; + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct blk_mq_ctxs *ctxs; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +struct blk_flush_queue { + unsigned int flush_queue_delayed:1; + unsigned int flush_pending_idx:1; + unsigned int flush_running_idx:1; + blk_status_t rq_status; + unsigned long flush_pending_since; + struct list_head flush_queue[2]; + struct list_head flush_data_in_flight; + struct request *flush_rq; + + /* + * flush_rq shares tag with this rq, both can't be active + * at the same time + */ + struct request *orig_rq; + spinlock_t mq_flush_lock; +}; + +//only kernel ali4000 >= 007 +static inline int enable_detect_flush_rq(void) +{ + return 1; +} + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue_type(struct request_queue *q, + enum hctx_type type, + unsigned int cpu) +{ + return q->queue_hw_ctx[q->tag_set->map[type].mq_map[cpu]]; +} + +/* + * blk_mq_map_queue() - map (cmd_flags,type) to hardware queue + * @q: request queue + * @flags: request command flags + * @cpu: CPU + */ +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + unsigned int flags, + unsigned int cpu) +{ + enum hctx_type type = HCTX_TYPE_DEFAULT; + + if ((flags & REQ_HIPRI) && + q->tag_set->nr_maps > HCTX_TYPE_POLL && + q->tag_set->map[HCTX_TYPE_POLL].nr_queues && + test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) + type = HCTX_TYPE_POLL; + + else if (((flags & REQ_OP_MASK) == REQ_OP_READ) && + q->tag_set->nr_maps > HCTX_TYPE_READ && + q->tag_set->map[HCTX_TYPE_READ].nr_queues) + type = HCTX_TYPE_READ; + + return blk_mq_map_queue_type(q, type, cpu); +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->cmd_flags, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? blk_mq_rq_from_pdu(data) : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return rq ? rq->internal_tag : -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->__deadline > rq->timeout) + return jiffies_to_usecs(rq->__deadline - rq->timeout) * 1000; + return 0; +} + +/* + * (ALIKERNEL && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 151)) || + * LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0) + */ +static inline u64 get_check_hang_time_ns(void) +{ + return ktime_get_ns(); +} + +//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +extern fn_queue_tag_busy_iter sym_blk_mq_queue_tag_busy_iter; +//#endif +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static blk_mq_rq_iter fn_blk_mq_check_hang = NULL; +static void blk_mq_check_rq_hang(struct blk_mq_hw_ctx *hctx, + struct request *rq, void *priv, bool reserved) +{ + if (fn_blk_mq_check_hang) + fn_blk_mq_check_hang(rq, priv, reserved); +} + +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + fn_blk_mq_check_hang = fn; +/* +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + if (!percpu_ref_tryget(&q->q_usage_counter)) + return -EBUSY; +#endif +*/ + sym_blk_mq_queue_tag_busy_iter(q, blk_mq_check_rq_hang, data); +/* +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + blk_queue_exit(q); +#endif +#else + struct blk_mq_hw_ctx *hctx; + int i; + + queue_for_each_hw_ctx(q, hctx, i) { + if (!(hctx->nr_ctx && hctx->tags)) + continue; + blk_mq_tag_busy_iter(hctx, blk_mq_check_rq_hang, data); + } +#endif +*/ + return 0; +} +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-23.4.an8.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-23.4.an8.x86_64/nvme.h new file mode 100644 index 00000000..35cb9aba --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-23.4.an8.x86_64/nvme.h @@ -0,0 +1,62 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 last_sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u8 polled; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-23.4.an8.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-23.4.an8.x86_64/virtio_blk.h new file mode 100644 index 00000000..e7793610 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-23.4.an8.x86_64/virtio_blk.h @@ -0,0 +1,292 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* Last written value to avail->flags */ + u16 avail_flags_shadow; + + /* Last written value to avail->idx in guest byte order */ + u16 avail_idx_shadow; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; + +struct virtio_blk { + /* + * This mutex must be held by anything that may run after + * virtblk_remove() sets vblk->vdev to NULL. + * + * blk-mq, virtqueue processing, and sysfs attribute code paths are + * shut down before vblk->vdev is set to NULL and therefore do not need + * to hold this mutex. + */ + struct mutex vdev_mutex; + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* + * Tracks references from block_device_operations open/release and + * virtio_driver probe/remove so this object can be freed once no + * longer in use. + */ + refcount_t refs; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { +#ifdef CONFIG_VIRTIO_BLK_SCSI + struct scsi_request sreq; /* for SCSI passthrough, must be first */ + u8 sense[SCSI_SENSE_BUFFERSIZE]; + struct virtio_scsi_inhdr in_hdr; +#endif + struct virtio_blk_outhdr out_hdr; + u8 status; + struct scatterlist sg[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctxs { + struct kobject kobj; + struct blk_mq_ctx __percpu *queue_ctx; +}; + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned short index_hw[HCTX_MAX_TYPES]; + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct blk_mq_ctxs *ctxs; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +struct blk_flush_queue { + unsigned int flush_queue_delayed:1; + unsigned int flush_pending_idx:1; + unsigned int flush_running_idx:1; + blk_status_t rq_status; + unsigned long flush_pending_since; + struct list_head flush_queue[2]; + struct list_head flush_data_in_flight; + struct request *flush_rq; + + /* + * flush_rq shares tag with this rq, both can't be active + * at the same time + */ + struct request *orig_rq; + spinlock_t mq_flush_lock; +}; + +//only kernel ali4000 >= 007 +static inline int enable_detect_flush_rq(void) +{ + return 1; +} + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue_type(struct request_queue *q, + enum hctx_type type, + unsigned int cpu) +{ + return q->queue_hw_ctx[q->tag_set->map[type].mq_map[cpu]]; +} + +/* + * blk_mq_map_queue() - map (cmd_flags,type) to hardware queue + * @q: request queue + * @flags: request command flags + * @cpu: CPU + */ +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + unsigned int flags, + unsigned int cpu) +{ + enum hctx_type type = HCTX_TYPE_DEFAULT; + + if ((flags & REQ_HIPRI) && + q->tag_set->nr_maps > HCTX_TYPE_POLL && + q->tag_set->map[HCTX_TYPE_POLL].nr_queues && + test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) + type = HCTX_TYPE_POLL; + + else if (((flags & REQ_OP_MASK) == REQ_OP_READ) && + q->tag_set->nr_maps > HCTX_TYPE_READ && + q->tag_set->map[HCTX_TYPE_READ].nr_queues) + type = HCTX_TYPE_READ; + + return blk_mq_map_queue_type(q, type, cpu); +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->cmd_flags, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? blk_mq_rq_from_pdu(data) : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return rq ? rq->internal_tag : -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->__deadline > rq->timeout) + return jiffies_to_usecs(rq->__deadline - rq->timeout) * 1000; + return 0; +} + +/* + * (ALIKERNEL && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 151)) || + * LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0) + */ +static inline u64 get_check_hang_time_ns(void) +{ + return ktime_get_ns(); +} + +//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +extern fn_queue_tag_busy_iter sym_blk_mq_queue_tag_busy_iter; +//#endif +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static blk_mq_rq_iter fn_blk_mq_check_hang = NULL; +static void blk_mq_check_rq_hang(struct blk_mq_hw_ctx *hctx, + struct request *rq, void *priv, bool reserved) +{ + if (fn_blk_mq_check_hang) + fn_blk_mq_check_hang(rq, priv, reserved); +} + +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + fn_blk_mq_check_hang = fn; +/* +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + if (!percpu_ref_tryget(&q->q_usage_counter)) + return -EBUSY; +#endif +*/ + sym_blk_mq_queue_tag_busy_iter(q, blk_mq_check_rq_hang, data); +/* +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + blk_queue_exit(q); +#endif +#else + struct blk_mq_hw_ctx *hctx; + int i; + + queue_for_each_hw_ctx(q, hctx, i) { + if (!(hctx->nr_ctx && hctx->tags)) + continue; + blk_mq_tag_busy_iter(hctx, blk_mq_check_rq_hang, data); + } +#endif +*/ + return 0; +} +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-23.al7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-23.al7.x86_64/nvme.h new file mode 100644 index 00000000..35cb9aba --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-23.al7.x86_64/nvme.h @@ -0,0 +1,62 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 last_sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u8 polled; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-23.al7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-23.al7.x86_64/virtio_blk.h new file mode 100644 index 00000000..e7793610 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-23.al7.x86_64/virtio_blk.h @@ -0,0 +1,292 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* Last written value to avail->flags */ + u16 avail_flags_shadow; + + /* Last written value to avail->idx in guest byte order */ + u16 avail_idx_shadow; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; + +struct virtio_blk { + /* + * This mutex must be held by anything that may run after + * virtblk_remove() sets vblk->vdev to NULL. + * + * blk-mq, virtqueue processing, and sysfs attribute code paths are + * shut down before vblk->vdev is set to NULL and therefore do not need + * to hold this mutex. + */ + struct mutex vdev_mutex; + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* + * Tracks references from block_device_operations open/release and + * virtio_driver probe/remove so this object can be freed once no + * longer in use. + */ + refcount_t refs; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { +#ifdef CONFIG_VIRTIO_BLK_SCSI + struct scsi_request sreq; /* for SCSI passthrough, must be first */ + u8 sense[SCSI_SENSE_BUFFERSIZE]; + struct virtio_scsi_inhdr in_hdr; +#endif + struct virtio_blk_outhdr out_hdr; + u8 status; + struct scatterlist sg[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctxs { + struct kobject kobj; + struct blk_mq_ctx __percpu *queue_ctx; +}; + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned short index_hw[HCTX_MAX_TYPES]; + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct blk_mq_ctxs *ctxs; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +struct blk_flush_queue { + unsigned int flush_queue_delayed:1; + unsigned int flush_pending_idx:1; + unsigned int flush_running_idx:1; + blk_status_t rq_status; + unsigned long flush_pending_since; + struct list_head flush_queue[2]; + struct list_head flush_data_in_flight; + struct request *flush_rq; + + /* + * flush_rq shares tag with this rq, both can't be active + * at the same time + */ + struct request *orig_rq; + spinlock_t mq_flush_lock; +}; + +//only kernel ali4000 >= 007 +static inline int enable_detect_flush_rq(void) +{ + return 1; +} + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue_type(struct request_queue *q, + enum hctx_type type, + unsigned int cpu) +{ + return q->queue_hw_ctx[q->tag_set->map[type].mq_map[cpu]]; +} + +/* + * blk_mq_map_queue() - map (cmd_flags,type) to hardware queue + * @q: request queue + * @flags: request command flags + * @cpu: CPU + */ +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + unsigned int flags, + unsigned int cpu) +{ + enum hctx_type type = HCTX_TYPE_DEFAULT; + + if ((flags & REQ_HIPRI) && + q->tag_set->nr_maps > HCTX_TYPE_POLL && + q->tag_set->map[HCTX_TYPE_POLL].nr_queues && + test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) + type = HCTX_TYPE_POLL; + + else if (((flags & REQ_OP_MASK) == REQ_OP_READ) && + q->tag_set->nr_maps > HCTX_TYPE_READ && + q->tag_set->map[HCTX_TYPE_READ].nr_queues) + type = HCTX_TYPE_READ; + + return blk_mq_map_queue_type(q, type, cpu); +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->cmd_flags, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? blk_mq_rq_from_pdu(data) : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return rq ? rq->internal_tag : -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->__deadline > rq->timeout) + return jiffies_to_usecs(rq->__deadline - rq->timeout) * 1000; + return 0; +} + +/* + * (ALIKERNEL && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 151)) || + * LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0) + */ +static inline u64 get_check_hang_time_ns(void) +{ + return ktime_get_ns(); +} + +//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +extern fn_queue_tag_busy_iter sym_blk_mq_queue_tag_busy_iter; +//#endif +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static blk_mq_rq_iter fn_blk_mq_check_hang = NULL; +static void blk_mq_check_rq_hang(struct blk_mq_hw_ctx *hctx, + struct request *rq, void *priv, bool reserved) +{ + if (fn_blk_mq_check_hang) + fn_blk_mq_check_hang(rq, priv, reserved); +} + +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + fn_blk_mq_check_hang = fn; +/* +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + if (!percpu_ref_tryget(&q->q_usage_counter)) + return -EBUSY; +#endif +*/ + sym_blk_mq_queue_tag_busy_iter(q, blk_mq_check_rq_hang, data); +/* +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + blk_queue_exit(q); +#endif +#else + struct blk_mq_hw_ctx *hctx; + int i; + + queue_for_each_hw_ctx(q, hctx, i) { + if (!(hctx->nr_ctx && hctx->tags)) + continue; + blk_mq_tag_busy_iter(hctx, blk_mq_check_rq_hang, data); + } +#endif +*/ + return 0; +} +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.1.al7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.1.al7.x86_64/nvme.h new file mode 100644 index 00000000..35cb9aba --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.1.al7.x86_64/nvme.h @@ -0,0 +1,62 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 last_sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u8 polled; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.1.al7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.1.al7.x86_64/virtio_blk.h new file mode 100644 index 00000000..e7793610 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.1.al7.x86_64/virtio_blk.h @@ -0,0 +1,292 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* Last written value to avail->flags */ + u16 avail_flags_shadow; + + /* Last written value to avail->idx in guest byte order */ + u16 avail_idx_shadow; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; + +struct virtio_blk { + /* + * This mutex must be held by anything that may run after + * virtblk_remove() sets vblk->vdev to NULL. + * + * blk-mq, virtqueue processing, and sysfs attribute code paths are + * shut down before vblk->vdev is set to NULL and therefore do not need + * to hold this mutex. + */ + struct mutex vdev_mutex; + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* + * Tracks references from block_device_operations open/release and + * virtio_driver probe/remove so this object can be freed once no + * longer in use. + */ + refcount_t refs; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { +#ifdef CONFIG_VIRTIO_BLK_SCSI + struct scsi_request sreq; /* for SCSI passthrough, must be first */ + u8 sense[SCSI_SENSE_BUFFERSIZE]; + struct virtio_scsi_inhdr in_hdr; +#endif + struct virtio_blk_outhdr out_hdr; + u8 status; + struct scatterlist sg[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctxs { + struct kobject kobj; + struct blk_mq_ctx __percpu *queue_ctx; +}; + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned short index_hw[HCTX_MAX_TYPES]; + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct blk_mq_ctxs *ctxs; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +struct blk_flush_queue { + unsigned int flush_queue_delayed:1; + unsigned int flush_pending_idx:1; + unsigned int flush_running_idx:1; + blk_status_t rq_status; + unsigned long flush_pending_since; + struct list_head flush_queue[2]; + struct list_head flush_data_in_flight; + struct request *flush_rq; + + /* + * flush_rq shares tag with this rq, both can't be active + * at the same time + */ + struct request *orig_rq; + spinlock_t mq_flush_lock; +}; + +//only kernel ali4000 >= 007 +static inline int enable_detect_flush_rq(void) +{ + return 1; +} + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue_type(struct request_queue *q, + enum hctx_type type, + unsigned int cpu) +{ + return q->queue_hw_ctx[q->tag_set->map[type].mq_map[cpu]]; +} + +/* + * blk_mq_map_queue() - map (cmd_flags,type) to hardware queue + * @q: request queue + * @flags: request command flags + * @cpu: CPU + */ +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + unsigned int flags, + unsigned int cpu) +{ + enum hctx_type type = HCTX_TYPE_DEFAULT; + + if ((flags & REQ_HIPRI) && + q->tag_set->nr_maps > HCTX_TYPE_POLL && + q->tag_set->map[HCTX_TYPE_POLL].nr_queues && + test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) + type = HCTX_TYPE_POLL; + + else if (((flags & REQ_OP_MASK) == REQ_OP_READ) && + q->tag_set->nr_maps > HCTX_TYPE_READ && + q->tag_set->map[HCTX_TYPE_READ].nr_queues) + type = HCTX_TYPE_READ; + + return blk_mq_map_queue_type(q, type, cpu); +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->cmd_flags, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? blk_mq_rq_from_pdu(data) : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return rq ? rq->internal_tag : -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->__deadline > rq->timeout) + return jiffies_to_usecs(rq->__deadline - rq->timeout) * 1000; + return 0; +} + +/* + * (ALIKERNEL && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 151)) || + * LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0) + */ +static inline u64 get_check_hang_time_ns(void) +{ + return ktime_get_ns(); +} + +//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +extern fn_queue_tag_busy_iter sym_blk_mq_queue_tag_busy_iter; +//#endif +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static blk_mq_rq_iter fn_blk_mq_check_hang = NULL; +static void blk_mq_check_rq_hang(struct blk_mq_hw_ctx *hctx, + struct request *rq, void *priv, bool reserved) +{ + if (fn_blk_mq_check_hang) + fn_blk_mq_check_hang(rq, priv, reserved); +} + +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + fn_blk_mq_check_hang = fn; +/* +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + if (!percpu_ref_tryget(&q->q_usage_counter)) + return -EBUSY; +#endif +*/ + sym_blk_mq_queue_tag_busy_iter(q, blk_mq_check_rq_hang, data); +/* +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + blk_queue_exit(q); +#endif +#else + struct blk_mq_hw_ctx *hctx; + int i; + + queue_for_each_hw_ctx(q, hctx, i) { + if (!(hctx->nr_ctx && hctx->tags)) + continue; + blk_mq_tag_busy_iter(hctx, blk_mq_check_rq_hang, data); + } +#endif +*/ + return 0; +} +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.8.an8.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.8.an8.x86_64/nvme.h new file mode 100644 index 00000000..35cb9aba --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.8.an8.x86_64/nvme.h @@ -0,0 +1,62 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 last_sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u8 polled; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.8.an8.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.8.an8.x86_64/virtio_blk.h new file mode 100644 index 00000000..e7793610 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.8.an8.x86_64/virtio_blk.h @@ -0,0 +1,292 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* Last written value to avail->flags */ + u16 avail_flags_shadow; + + /* Last written value to avail->idx in guest byte order */ + u16 avail_idx_shadow; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; + +struct virtio_blk { + /* + * This mutex must be held by anything that may run after + * virtblk_remove() sets vblk->vdev to NULL. + * + * blk-mq, virtqueue processing, and sysfs attribute code paths are + * shut down before vblk->vdev is set to NULL and therefore do not need + * to hold this mutex. + */ + struct mutex vdev_mutex; + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* + * Tracks references from block_device_operations open/release and + * virtio_driver probe/remove so this object can be freed once no + * longer in use. + */ + refcount_t refs; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { +#ifdef CONFIG_VIRTIO_BLK_SCSI + struct scsi_request sreq; /* for SCSI passthrough, must be first */ + u8 sense[SCSI_SENSE_BUFFERSIZE]; + struct virtio_scsi_inhdr in_hdr; +#endif + struct virtio_blk_outhdr out_hdr; + u8 status; + struct scatterlist sg[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctxs { + struct kobject kobj; + struct blk_mq_ctx __percpu *queue_ctx; +}; + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned short index_hw[HCTX_MAX_TYPES]; + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct blk_mq_ctxs *ctxs; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +struct blk_flush_queue { + unsigned int flush_queue_delayed:1; + unsigned int flush_pending_idx:1; + unsigned int flush_running_idx:1; + blk_status_t rq_status; + unsigned long flush_pending_since; + struct list_head flush_queue[2]; + struct list_head flush_data_in_flight; + struct request *flush_rq; + + /* + * flush_rq shares tag with this rq, both can't be active + * at the same time + */ + struct request *orig_rq; + spinlock_t mq_flush_lock; +}; + +//only kernel ali4000 >= 007 +static inline int enable_detect_flush_rq(void) +{ + return 1; +} + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue_type(struct request_queue *q, + enum hctx_type type, + unsigned int cpu) +{ + return q->queue_hw_ctx[q->tag_set->map[type].mq_map[cpu]]; +} + +/* + * blk_mq_map_queue() - map (cmd_flags,type) to hardware queue + * @q: request queue + * @flags: request command flags + * @cpu: CPU + */ +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + unsigned int flags, + unsigned int cpu) +{ + enum hctx_type type = HCTX_TYPE_DEFAULT; + + if ((flags & REQ_HIPRI) && + q->tag_set->nr_maps > HCTX_TYPE_POLL && + q->tag_set->map[HCTX_TYPE_POLL].nr_queues && + test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) + type = HCTX_TYPE_POLL; + + else if (((flags & REQ_OP_MASK) == REQ_OP_READ) && + q->tag_set->nr_maps > HCTX_TYPE_READ && + q->tag_set->map[HCTX_TYPE_READ].nr_queues) + type = HCTX_TYPE_READ; + + return blk_mq_map_queue_type(q, type, cpu); +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->cmd_flags, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? blk_mq_rq_from_pdu(data) : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return rq ? rq->internal_tag : -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->__deadline > rq->timeout) + return jiffies_to_usecs(rq->__deadline - rq->timeout) * 1000; + return 0; +} + +/* + * (ALIKERNEL && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 151)) || + * LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0) + */ +static inline u64 get_check_hang_time_ns(void) +{ + return ktime_get_ns(); +} + +//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +extern fn_queue_tag_busy_iter sym_blk_mq_queue_tag_busy_iter; +//#endif +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static blk_mq_rq_iter fn_blk_mq_check_hang = NULL; +static void blk_mq_check_rq_hang(struct blk_mq_hw_ctx *hctx, + struct request *rq, void *priv, bool reserved) +{ + if (fn_blk_mq_check_hang) + fn_blk_mq_check_hang(rq, priv, reserved); +} + +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + fn_blk_mq_check_hang = fn; +/* +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + if (!percpu_ref_tryget(&q->q_usage_counter)) + return -EBUSY; +#endif +*/ + sym_blk_mq_queue_tag_busy_iter(q, blk_mq_check_rq_hang, data); +/* +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + blk_queue_exit(q); +#endif +#else + struct blk_mq_hw_ctx *hctx; + int i; + + queue_for_each_hw_ctx(q, hctx, i) { + if (!(hctx->nr_ctx && hctx->tags)) + continue; + blk_mq_tag_busy_iter(hctx, blk_mq_check_rq_hang, data); + } +#endif +*/ + return 0; +} +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.al7.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.al7.x86_64/nvme.h new file mode 100644 index 00000000..35cb9aba --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.al7.x86_64/nvme.h @@ -0,0 +1,62 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 last_sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u8 polled; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.al7.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.al7.x86_64/virtio_blk.h new file mode 100644 index 00000000..e7793610 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-24.al7.x86_64/virtio_blk.h @@ -0,0 +1,292 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* Last written value to avail->flags */ + u16 avail_flags_shadow; + + /* Last written value to avail->idx in guest byte order */ + u16 avail_idx_shadow; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; + +struct virtio_blk { + /* + * This mutex must be held by anything that may run after + * virtblk_remove() sets vblk->vdev to NULL. + * + * blk-mq, virtqueue processing, and sysfs attribute code paths are + * shut down before vblk->vdev is set to NULL and therefore do not need + * to hold this mutex. + */ + struct mutex vdev_mutex; + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* + * Tracks references from block_device_operations open/release and + * virtio_driver probe/remove so this object can be freed once no + * longer in use. + */ + refcount_t refs; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { +#ifdef CONFIG_VIRTIO_BLK_SCSI + struct scsi_request sreq; /* for SCSI passthrough, must be first */ + u8 sense[SCSI_SENSE_BUFFERSIZE]; + struct virtio_scsi_inhdr in_hdr; +#endif + struct virtio_blk_outhdr out_hdr; + u8 status; + struct scatterlist sg[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctxs { + struct kobject kobj; + struct blk_mq_ctx __percpu *queue_ctx; +}; + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned short index_hw[HCTX_MAX_TYPES]; + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct blk_mq_ctxs *ctxs; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +struct blk_flush_queue { + unsigned int flush_queue_delayed:1; + unsigned int flush_pending_idx:1; + unsigned int flush_running_idx:1; + blk_status_t rq_status; + unsigned long flush_pending_since; + struct list_head flush_queue[2]; + struct list_head flush_data_in_flight; + struct request *flush_rq; + + /* + * flush_rq shares tag with this rq, both can't be active + * at the same time + */ + struct request *orig_rq; + spinlock_t mq_flush_lock; +}; + +//only kernel ali4000 >= 007 +static inline int enable_detect_flush_rq(void) +{ + return 1; +} + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue_type(struct request_queue *q, + enum hctx_type type, + unsigned int cpu) +{ + return q->queue_hw_ctx[q->tag_set->map[type].mq_map[cpu]]; +} + +/* + * blk_mq_map_queue() - map (cmd_flags,type) to hardware queue + * @q: request queue + * @flags: request command flags + * @cpu: CPU + */ +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + unsigned int flags, + unsigned int cpu) +{ + enum hctx_type type = HCTX_TYPE_DEFAULT; + + if ((flags & REQ_HIPRI) && + q->tag_set->nr_maps > HCTX_TYPE_POLL && + q->tag_set->map[HCTX_TYPE_POLL].nr_queues && + test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) + type = HCTX_TYPE_POLL; + + else if (((flags & REQ_OP_MASK) == REQ_OP_READ) && + q->tag_set->nr_maps > HCTX_TYPE_READ && + q->tag_set->map[HCTX_TYPE_READ].nr_queues) + type = HCTX_TYPE_READ; + + return blk_mq_map_queue_type(q, type, cpu); +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->cmd_flags, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? blk_mq_rq_from_pdu(data) : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return rq ? rq->internal_tag : -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->__deadline > rq->timeout) + return jiffies_to_usecs(rq->__deadline - rq->timeout) * 1000; + return 0; +} + +/* + * (ALIKERNEL && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 151)) || + * LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0) + */ +static inline u64 get_check_hang_time_ns(void) +{ + return ktime_get_ns(); +} + +//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +extern fn_queue_tag_busy_iter sym_blk_mq_queue_tag_busy_iter; +//#endif +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static blk_mq_rq_iter fn_blk_mq_check_hang = NULL; +static void blk_mq_check_rq_hang(struct blk_mq_hw_ctx *hctx, + struct request *rq, void *priv, bool reserved) +{ + if (fn_blk_mq_check_hang) + fn_blk_mq_check_hang(rq, priv, reserved); +} + +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + fn_blk_mq_check_hang = fn; +/* +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + if (!percpu_ref_tryget(&q->q_usage_counter)) + return -EBUSY; +#endif +*/ + sym_blk_mq_queue_tag_busy_iter(q, blk_mq_check_rq_hang, data); +/* +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + blk_queue_exit(q); +#endif +#else + struct blk_mq_hw_ctx *hctx; + int i; + + queue_for_each_hw_ctx(q, hctx, i) { + if (!(hctx->nr_ctx && hctx->tags)) + continue; + blk_mq_tag_busy_iter(hctx, blk_mq_check_rq_hang, data); + } +#endif +*/ + return 0; +} +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-25.an8.x86_64/nvme.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-25.an8.x86_64/nvme.h new file mode 100644 index 00000000..35cb9aba --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-25.an8.x86_64/nvme.h @@ -0,0 +1,62 @@ +#ifndef _NVME_H +#define _NVME_H + +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +struct nvme_queue { + struct device *q_dmadev; + void *nvme_dev; //struct nvme_dev *dev; + spinlock_t sq_lock; + struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; + spinlock_t cq_lock ____cacheline_aligned_in_smp; + volatile struct nvme_completion *cqes; + struct blk_mq_tags **tags; + dma_addr_t sq_dma_addr; + dma_addr_t cq_dma_addr; + u32 __iomem *q_db; + u16 q_depth; + s16 cq_vector; + u16 sq_tail; + u16 last_sq_tail; + u16 cq_head; + u16 last_cq_head; + u16 qid; + u8 cq_phase; + u8 polled; + u32 *dbbuf_sq_db; + u32 *dbbuf_cq_db; + u32 *dbbuf_sq_ei; + u32 *dbbuf_cq_ei; +}; + +static int get_sq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int tail = nvmeq->sq_tail; + struct nvme_command cmd; + + do { + if (nvmeq->sq_cmds_io) { + memcpy_toio(&cmd, &nvmeq->sq_cmds_io[tail], sizeof(struct nvme_command)); + if (cmd.common.command_id == rq->tag) + return tail; + } + else if (nvmeq->sq_cmds[tail].common.command_id == rq->tag) + return tail; + } while (--tail >= 0); + return -1; +} + +static unsigned long get_cmd_ctx(struct nvme_queue *nvmeq, struct request *rq) +{ + //struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + return 0; +} +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-25.an8.x86_64/virtio_blk.h b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-25.an8.x86_64/virtio_blk.h new file mode 100644 index 00000000..e7793610 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/include/4.19.91-25.an8.x86_64/virtio_blk.h @@ -0,0 +1,292 @@ +#ifndef _VIRTIO_BLK_H +#define _VIRTIO_BLK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +#define VQ_NAME_LEN 16 + +struct virtio_blk_vq { + struct virtqueue *vq; + spinlock_t lock; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +struct vring_desc_state { + void *data; /* Data for callback. */ + struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ +}; + +struct vring_virtqueue { + struct virtqueue vq; + + /* Actual memory layout for this queue */ + struct vring vring; + + /* Can we use weak barriers? */ + bool weak_barriers; + + /* Other side has made a mess, don't try any more. */ + bool broken; + + /* Host supports indirect buffers */ + bool indirect; + + /* Host publishes avail event idx */ + bool event; + + /* Head of free buffer list. */ + unsigned int free_head; + /* Number we've added since last sync. */ + unsigned int num_added; + + /* Last used index we've seen. */ + u16 last_used_idx; + + /* Last written value to avail->flags */ + u16 avail_flags_shadow; + + /* Last written value to avail->idx in guest byte order */ + u16 avail_idx_shadow; + + /* How to notify other side. FIXME: commonalize hcalls! */ + bool (*notify)(struct virtqueue *vq); + + /* DMA, allocation, and size information */ + bool we_own_ring; + size_t queue_size_in_bytes; + dma_addr_t queue_dma_addr; + +#ifdef DEBUG + /* They're supposed to lock for us. */ + unsigned int in_use; + + /* Figure out if their kicks are too delayed. */ + bool last_add_time_valid; + ktime_t last_add_time; +#endif + + /* Per-descriptor state. */ + struct vring_desc_state desc_state[]; +}; + +struct virtio_blk { + /* + * This mutex must be held by anything that may run after + * virtblk_remove() sets vblk->vdev to NULL. + * + * blk-mq, virtqueue processing, and sysfs attribute code paths are + * shut down before vblk->vdev is set to NULL and therefore do not need + * to hold this mutex. + */ + struct mutex vdev_mutex; + struct virtio_device *vdev; + + /* The disk structure for the kernel. */ + struct gendisk *disk; + + /* Block layer tags. */ + struct blk_mq_tag_set tag_set; + + /* Process context for config space updates */ + struct work_struct config_work; + + /* + * Tracks references from block_device_operations open/release and + * virtio_driver probe/remove so this object can be freed once no + * longer in use. + */ + refcount_t refs; + + /* What host tells us, plus 2 for header & tailer. */ + unsigned int sg_elems; + + /* Ida index - used to track minor number allocations. */ + int index; + + /* num of vqs */ + int num_vqs; + struct virtio_blk_vq *vqs; +}; + +struct virtblk_req { +#ifdef CONFIG_VIRTIO_BLK_SCSI + struct scsi_request sreq; /* for SCSI passthrough, must be first */ + u8 sense[SCSI_SENSE_BUFFERSIZE]; + struct virtio_scsi_inhdr in_hdr; +#endif + struct virtio_blk_outhdr out_hdr; + u8 status; + struct scatterlist sg[]; +}; +#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) + +struct blk_mq_ctxs { + struct kobject kobj; + struct blk_mq_ctx __percpu *queue_ctx; +}; + +struct blk_mq_ctx { + struct { + spinlock_t lock; + struct list_head rq_list; + } ____cacheline_aligned_in_smp; + + unsigned int cpu; + unsigned short index_hw[HCTX_MAX_TYPES]; + /* incremented at dispatch time */ + unsigned long rq_dispatched[2]; + unsigned long rq_merged; + + /* incremented at completion time */ + unsigned long ____cacheline_aligned_in_smp rq_completed[2]; + + struct request_queue *queue; + struct blk_mq_ctxs *ctxs; + struct kobject kobj; +} ____cacheline_aligned_in_smp; + +struct blk_flush_queue { + unsigned int flush_queue_delayed:1; + unsigned int flush_pending_idx:1; + unsigned int flush_running_idx:1; + blk_status_t rq_status; + unsigned long flush_pending_since; + struct list_head flush_queue[2]; + struct list_head flush_data_in_flight; + struct request *flush_rq; + + /* + * flush_rq shares tag with this rq, both can't be active + * at the same time + */ + struct request *orig_rq; + spinlock_t mq_flush_lock; +}; + +//only kernel ali4000 >= 007 +static inline int enable_detect_flush_rq(void) +{ + return 1; +} + +static inline struct blk_mq_hw_ctx *blk_mq_map_queue_type(struct request_queue *q, + enum hctx_type type, + unsigned int cpu) +{ + return q->queue_hw_ctx[q->tag_set->map[type].mq_map[cpu]]; +} + +/* + * blk_mq_map_queue() - map (cmd_flags,type) to hardware queue + * @q: request queue + * @flags: request command flags + * @cpu: CPU + */ +static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, + unsigned int flags, + unsigned int cpu) +{ + enum hctx_type type = HCTX_TYPE_DEFAULT; + + if ((flags & REQ_HIPRI) && + q->tag_set->nr_maps > HCTX_TYPE_POLL && + q->tag_set->map[HCTX_TYPE_POLL].nr_queues && + test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) + type = HCTX_TYPE_POLL; + + else if (((flags & REQ_OP_MASK) == REQ_OP_READ) && + q->tag_set->nr_maps > HCTX_TYPE_READ && + q->tag_set->map[HCTX_TYPE_READ].nr_queues) + type = HCTX_TYPE_READ; + + return blk_mq_map_queue_type(q, type, cpu); +} + +static inline struct blk_mq_hw_ctx *blk_mq_get_hctx_byrq(struct request *rq) +{ + return blk_mq_map_queue(rq->q, rq->cmd_flags, rq->mq_ctx->cpu); +} + +static inline struct request *desc_state_data_to_req(struct virtqueue *vq, int head) +{ + void *data = to_vvq(vq)->desc_state[head].data; + return data ? blk_mq_rq_from_pdu(data) : NULL; +} + +static inline int get_rq_internal_tag(struct request *rq) +{ + return rq ? rq->internal_tag : -1; +} + +static inline unsigned long get_issue_driver_ns(struct request *rq) +{ + if (!rq) + return 0; + if (rq->io_start_time_ns) + return rq->io_start_time_ns; + if (rq->__deadline > rq->timeout) + return jiffies_to_usecs(rq->__deadline - rq->timeout) * 1000; + return 0; +} + +/* + * (ALIKERNEL && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 151)) || + * LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0) + */ +static inline u64 get_check_hang_time_ns(void) +{ + return ktime_get_ns(); +} + +//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +extern fn_queue_tag_busy_iter sym_blk_mq_queue_tag_busy_iter; +//#endif +typedef void (*blk_mq_rq_iter)(struct request *, void *, bool); +static blk_mq_rq_iter fn_blk_mq_check_hang = NULL; +static void blk_mq_check_rq_hang(struct blk_mq_hw_ctx *hctx, + struct request *rq, void *priv, bool reserved) +{ + if (fn_blk_mq_check_hang) + fn_blk_mq_check_hang(rq, priv, reserved); +} + +static inline int iter_all_rq(struct request_queue *q, blk_mq_rq_iter fn, void *data) +{ + fn_blk_mq_check_hang = fn; +/* +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + if (!percpu_ref_tryget(&q->q_usage_counter)) + return -EBUSY; +#endif +*/ + sym_blk_mq_queue_tag_busy_iter(q, blk_mq_check_rq_hang, data); +/* +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) + blk_queue_exit(q); +#endif +#else + struct blk_mq_hw_ctx *hctx; + int i; + + queue_for_each_hw_ctx(q, hctx, i) { + if (!(hctx->nr_ctx && hctx->tags)) + continue; + blk_mq_tag_busy_iter(hctx, blk_mq_check_rq_hang, data); + } +#endif +*/ + return 0; +} +void get_vq_info(struct vq_info *vq_i, struct request *rq); +#endif diff --git a/source/lib/internal/kernel_module/modules/iosdiag/iosdiag.c b/source/lib/internal/kernel_module/modules/iosdiag/iosdiag.c new file mode 100644 index 00000000..c8efa55b --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/iosdiag.c @@ -0,0 +1,341 @@ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" +#include + +#define DISKHANG_DIR_NAME "disk_hang" + +#define DEFINE_PROC_ATTRIBUTE(name, __write, __mmap) \ + static int name##_open(struct inode *inode, struct file *file) \ + { \ + return single_open(file, name##_show, PDE_DATA(inode)); \ + } \ + static const struct file_operations name##_fops = { \ + .owner = THIS_MODULE, \ + .open = name##_open, \ + .read = seq_read, \ + .write = __write, \ + .mmap = __mmap, \ + .llseek = seq_lseek, \ + .release = single_release, \ + } + +#define DEFINE_PROC_ATTRIBUTE_RW(name) \ + static ssize_t name##_write(struct file *file, \ + const char __user *buf, \ + size_t count, loff_t *ppos) \ + { \ + return name##_store(PDE_DATA(file_inode(file)), buf, \ + count); \ + } \ + DEFINE_PROC_ATTRIBUTE(name, name##_write, name##_mmap) + +static DEFINE_MUTEX(rq_hang_buffer_mutex); + +struct rq_store { + struct list_head list; + struct request *rq; +}; +static struct rq_store g_rq_store[MAX_STORE_RQ_CNT]; +static struct rq_hang_info *g_rq_hang_info; +static int g_rq_hang_idx; +static unsigned long long g_rq_hang_total; +static int g_disk_type = -1; +static int g_bio_file_info; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +fn_queue_tag_busy_iter sym_blk_mq_queue_tag_busy_iter = NULL; +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0) +typedef void (*fn_mq_free_request)(struct request *rq); +static fn_mq_free_request sym_blk_mq_free_request; +#endif +fn_get_files_struct sym_get_files_struct = NULL; +fn_put_files_struct sym_put_files_struct = NULL; + +static void set_disk_type(char *buf) +{ + if (buf[0] == 'v' && buf[1] == 'd' && (buf[2] >= 'a' && buf[2] <= 'z')) + g_disk_type = DISK_VIRTIO_BLK; + else if (buf[0] == 's' && buf[1] == 'd' && (buf[2] >= 'a' && buf[2] <= 'z')) + g_disk_type = DISK_SCSI; + else if (!strncmp(buf, "nvme", 4)) + g_disk_type = DISK_NVME; + else + g_disk_type = -1; +} + +static int get_disk_type(void) +{ + return g_disk_type; +} + +int get_bio_file_info(void) +{ + return g_bio_file_info; +} + +static void store_hang_rq(struct request *rq, unsigned long long now) +{ + int index; + + if (g_rq_hang_idx >= MAX_STORE_RQ_CNT) + return; + + g_rq_hang_total++; + index = g_rq_hang_idx; + if (fill_hang_info_from_rq(&g_rq_hang_info[index], rq, + get_disk_type())) + return; + g_rq_hang_info[index].check_hang_ns = now; + g_rq_hang_info[index].req_addr = (unsigned long)rq; + g_rq_hang_idx++; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0) +static int is_flush_rq(struct request *rq) +{ + struct blk_mq_hw_ctx *hctx = blk_mq_get_hctx_byrq(rq); + + if (hctx && hctx->fq) + return hctx->fq->flush_rq == rq; + return 0; +} +#endif + +static void mq_check_rq_hang(struct request *rq, void *priv, bool reserved) +{ + int rq_hang_threshold = *((int *)priv); + u64 now = get_check_hang_time_ns(); + u64 duration; + + if (!rq) + return; + + if (g_rq_hang_idx >= MAX_STORE_RQ_CNT) + return; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0) + //if (is_flush_rq(rq) && !enable_detect_flush_rq()) + if (is_flush_rq(rq)) + return; + if (!refcount_inc_not_zero(&rq->ref)) + return; +#else + if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) + return; +#endif + duration = div_u64(now - rq->start_time_ns, NSEC_PER_MSEC); + if (duration >= rq_hang_threshold) + store_hang_rq(rq, now); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0) + //if (is_flush_rq(rq) && rq->end_io) + // rq->end_io(rq, 0); + //else if (refcount_dec_and_test(&rq->ref)) + if (refcount_dec_and_test(&rq->ref)) + sym_blk_mq_free_request(rq); +#endif +} + +static int sq_check_rq_hang(struct request_queue *q, int rq_hang_threshold) +{ + u64 now = get_check_hang_time_ns(); + u64 duration; + unsigned long flags; + struct request *rq, *tmp; + LIST_HEAD(rq_list); + int rq_store_idx = 0; + + spin_lock_irqsave(q->queue_lock, flags); + list_for_each_entry_safe(rq, tmp, &q->queue_head, queuelist) { + duration = div_u64(now - rq->start_time_ns, NSEC_PER_MSEC); + if (duration >= rq_hang_threshold && rq_store_idx < MAX_STORE_RQ_CNT) { + g_rq_store[rq_store_idx].rq = rq; + INIT_LIST_HEAD(&g_rq_store[rq_store_idx].list); + list_add(&g_rq_store[rq_store_idx].list, &rq_list); + rq_store_idx++; + } else + continue; + } + spin_unlock_irqrestore(q->queue_lock, flags); + + spin_lock_irqsave(q->queue_lock, flags); + list_for_each_entry_safe(rq, tmp, &q->timeout_list, timeout_list) { + duration = div_u64(now - rq->start_time_ns, NSEC_PER_MSEC); + if (duration >= rq_hang_threshold && rq_store_idx < MAX_STORE_RQ_CNT) { + g_rq_store[rq_store_idx].rq = rq; + INIT_LIST_HEAD(&g_rq_store[rq_store_idx].list); + list_add(&g_rq_store[rq_store_idx].list, &rq_list); + rq_store_idx++; + } else + continue; + } + spin_unlock_irqrestore(q->queue_lock, flags); + while(!list_empty(&rq_list)) { + struct rq_store *rqs; + rqs = list_first_entry(&rq_list, struct rq_store, list); + if (rqs->rq) + store_hang_rq(rqs->rq, now); + list_del_init(&rqs->list); + } + return 0; +} + +static int rq_hang_detect(dev_t devnum, int rq_hang_threshold) +{ + int ret = 0; + struct request_queue *q; + struct block_device *bdev; + + if (!devnum || rq_hang_threshold <= 0) + return -EINVAL; + + if (!(bdev = bdget(devnum))) { + printk("error: invalid devnum(%d:%d)\n", MAJOR(devnum), MINOR(devnum)); + return -EFAULT; + } + if (!bdev->bd_queue) { + if (!bdev->bd_disk || !(q = bdev_get_queue(bdev))) { + printk("error: can't get request queue for devnum(%d:%d)\n", + MAJOR(devnum), MINOR(devnum)); + bdput(bdev); + return -EFAULT; + } + } else + q = bdev->bd_queue; + + if (q->mq_ops) + ret = iter_all_rq(q, mq_check_rq_hang, &rq_hang_threshold); + else + ret = sq_check_rq_hang(q, rq_hang_threshold); + bdput(bdev); + return ret; +} + +static int rq_hang_show(struct seq_file *m, void *ptr) +{ + seq_printf(m, "total_rq_hang:%llu\n", g_rq_hang_total); + return 0; +} + +static ssize_t rq_hang_store(struct file *file, + const char __user *buf, size_t count) +{ + int ret; + char *p; + char chr[256]; + char diskname[BDEVNAME_SIZE] = {0}; + int major, minor; + int threshold = 0; + + if (count < 1) + return -EINVAL; + + if (copy_from_user(chr, buf, 256)) + return -EFAULT; + + /* echo "vdb:253:16 1000" > /proc/xxxxx */ + if (p = strstr(chr, ":")) { + memcpy(diskname, chr, (p - chr)); + ret = sscanf(p+1, "%d:%d %d %d", &major, &minor, &threshold, &g_bio_file_info); + if (ret < 3 || threshold <= 0 || major < 1 || minor < 0) { + printk("invalid argument \'%s\'\n", chr); + return -EINVAL; + } + } else { + printk("invalid argument \'%s\'\n", chr); + return -EINVAL; + } + mutex_lock(&rq_hang_buffer_mutex); + set_disk_type(diskname); + g_rq_hang_idx = 0; + memset(g_rq_hang_info, 0x0, sizeof(struct rq_hang_info) * MAX_STORE_RQ_CNT); + ret = rq_hang_detect(MKDEV(major, minor), threshold); + mutex_unlock(&rq_hang_buffer_mutex); + return ret ? ret : count; +} + +static int rq_hang_mmap(struct file *file, struct vm_area_struct *vma) +{ + return remap_vmalloc_range(vma, (void *)g_rq_hang_info, vma->vm_pgoff); +} +DEFINE_PROC_ATTRIBUTE_RW(rq_hang); + +int disk_hang_init(void) +{ + int ret; + struct proc_dir_entry *disk_hang_dir = NULL; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) + sym_blk_mq_queue_tag_busy_iter = + (fn_queue_tag_busy_iter)kallsyms_lookup_name("blk_mq_queue_tag_busy_iter"); + if (!sym_blk_mq_queue_tag_busy_iter) { + pr_err("not found symbol \"blk_mq_queue_tag_busy_iter\"\n"); + return -EFAULT; + } +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0) + sym_blk_mq_free_request = + (fn_mq_free_request)kallsyms_lookup_name("__blk_mq_free_request"); + if (!sym_blk_mq_free_request) { + pr_err("not found symbol \"__blk_mq_free_request\"\n"); + return -EFAULT; + } +#endif + sym_get_files_struct = + (fn_get_files_struct)kallsyms_lookup_name("get_files_struct"); + if (!sym_get_files_struct) + pr_warn("not found symbol \"get_files_struct\"\n"); + + sym_put_files_struct = + (fn_put_files_struct)kallsyms_lookup_name("put_files_struct"); + if (!sym_put_files_struct) + pr_warn("not found symbol \"put_files_struct\"\n"); + + disk_hang_dir = proc_mkdir(DISKHANG_DIR_NAME, NULL); + if (!disk_hang_dir) { + pr_err("create \"/proc/%s\" fail\n", DISKHANG_DIR_NAME); + return -ENOMEM; + } + if (!proc_create_data("rq_hang_detect", 0600, disk_hang_dir, + &rq_hang_fops, NULL)) { + pr_err("create \"/proc/%s/rq_hang_detect\" fail\n", + DISKHANG_DIR_NAME); + ret = -ENOMEM; + goto remove_proc; + } + g_rq_hang_info = vmalloc_user(sizeof(struct rq_hang_info) * MAX_STORE_RQ_CNT); + if (!g_rq_hang_info) { + pr_err("alloc memory \"rq hang info buffer\" fail\n"); + ret = -ENOMEM; + goto remove_proc; + } + memset(g_rq_hang_info, 0x0, sizeof(struct rq_hang_info) * MAX_STORE_RQ_CNT); + pr_info("iosdiag load success\n"); + return 0; +remove_proc: + remove_proc_subtree(DISKHANG_DIR_NAME, NULL); + return ret; +} + +int disk_hang_exit(void) +{ + if (g_rq_hang_info) { + vfree(g_rq_hang_info); + g_rq_hang_info = NULL; + } + remove_proc_subtree(DISKHANG_DIR_NAME, NULL); + return 0; +} +//module_init(disk_hang_init); +//module_exit(disk_hang_exit); +MODULE_LICENSE("GPL"); + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/iosdiag.h b/source/lib/internal/kernel_module/modules/iosdiag/iosdiag.h new file mode 100644 index 00000000..584c1b03 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/iosdiag.h @@ -0,0 +1,101 @@ +#ifndef __IOSDIAG_H +#define __IOSDIAG_H +#include +#include + +#define MAX_STORE_RQ_CNT 128 +#define MAX_FILE_NAME_LEN 255 +#define BIO_INFO_MAX_PAGES 32 +#define MAX_REQ_BIOS 32 + +enum disk_type { + DISK_VIRTIO_BLK, + DISK_NVME, + DISK_SCSI, +}; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0) +enum rq_atomic_flags { + REQ_ATOM_COMPLETE = 0, + REQ_ATOM_STARTED, +}; +#endif +/* +struct rq_buffer { + struct request rq; + unsigned long long check_time_ns; + void *rq_addr; +}; +*/ +struct vq_info { + int qid; + int vring_num; + int last_used_idx; + int used_idx; + int used_ring_flags; + int last_avail_idx; + int avail_idx; + int avail_ring_flags; + int event; + int rq_avail_idx; + int last_kick_avail_idx; + int rq_used_idx; +}; + +struct nvme_info { + int qid; + int q_depth; //sq/cq depth + int cq_head; //nvmeq->cqes[cq_head]~nvmeq->cqes[cq_end], including req->tag? + int cq_end; + int cq_rq_idx; //rq idx in cq + //int last_cq_head; //nvmeq->sq_head or nvmeq->last_cq_head + int sq_tail; //0~nvmeq->sq_cmds[idx].command_id, including req->tag? + int sq_rq_idx; //rq idx in sq + int sq_last_db; //last sq idx host kick nvme, nvmeq->q_db + unsigned long cmd_ctx; +}; + +struct scsi_info { + int done_hander_defined; + int is_mq; +}; + +struct bio_info { + unsigned long bio_addr; + unsigned long sector; + unsigned int size; + unsigned int pid; + char comm[TASK_COMM_LEN]; + char filename[MAX_FILE_NAME_LEN]; +}; + +struct rq_hang_info { + unsigned int data_len; + unsigned long sector; + unsigned long req_addr; + unsigned long long io_start_ns; + unsigned long long io_issue_driver_ns; + unsigned long long check_hang_ns; + char op[64]; + char state[16]; + struct vq_info vq; + struct nvme_info nvme; + struct scsi_info scsi; + int tag; + int internal_tag; + int cpu; + char diskname[BDEVNAME_SIZE]; + //int errors; + //unsigned long cmd_flags; + struct bio_info first_bio; +}; + +typedef void (*fn_queue_tag_busy_iter)(struct request_queue *q, busy_iter_fn *fn, void *priv); +typedef struct files_struct *(*fn_get_files_struct)(struct task_struct *); +typedef void (*fn_put_files_struct)(struct files_struct *fs); + +int fill_hang_info_from_rq(struct rq_hang_info *rq_hang_info, + struct request *rq, + int disk_type); +#endif + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/nvme.c b/source/lib/internal/kernel_module/modules/iosdiag/nvme.c new file mode 100644 index 00000000..b890579b --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/nvme.c @@ -0,0 +1,65 @@ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct nvme_queue *get_nvme_queue_by_rq(struct request *rq) +{ + struct blk_mq_hw_ctx *hctx; + + if (!rq) + return NULL; + + hctx = blk_mq_get_hctx_byrq(rq); + if (!hctx) + return NULL; + return hctx->driver_data ? hctx->driver_data : NULL; +} + +static int get_cq_end(struct nvme_queue *nvmeq, struct request *rq) +{ + int head = nvmeq->cq_head; + + do { + if (nvmeq->cqes[head].command_id == -1) + return head; + } while (++head < nvmeq->q_depth); + return -1; +} + +static int get_cq_rq_idx(struct nvme_queue *nvmeq, struct request *rq) +{ + int head = 0; + + do { + if (nvmeq->cqes[head].command_id == rq->tag) + return head; + } while (++head < nvmeq->q_depth); + return -1; +} + +void get_nvme_info(struct nvme_info *nvme_i, struct request *rq) +{ + struct nvme_queue *nvmeq; + + if (!(nvmeq = get_nvme_queue_by_rq(rq))) + return; + + nvme_i->qid = nvmeq->qid; + nvme_i->q_depth = nvmeq->q_depth; + nvme_i->cq_head = nvmeq->cq_head; + nvme_i->cq_end = get_cq_end(nvmeq, rq); + nvme_i->cq_rq_idx = get_cq_rq_idx(nvmeq, rq); + nvme_i->sq_tail = nvmeq->sq_tail; + nvme_i->sq_rq_idx = get_sq_rq_idx(nvmeq, rq); + nvme_i->sq_last_db = readl(nvmeq->q_db); + nvme_i->cmd_ctx = get_cmd_ctx(nvmeq, rq); +} + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/rq_hang.c b/source/lib/internal/kernel_module/modules/iosdiag/rq_hang.c new file mode 100644 index 00000000..fb4f56a2 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/rq_hang.c @@ -0,0 +1,322 @@ + +#include +#include +#include +#include +#include +#include +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) +#include +#endif +#include +#include +#include +#include +#include "iosdiag.h" +#include +#include + +struct req_op_name{ + int op; + char *op_str; +}; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0) +#define REQ_OP_NAME(name) {REQ_OP_##name, #name} +#else +#define REQ_READ 0 +#define REQ_OP_NAME(name) {REQ_##name, #name} +#endif +static struct req_op_name g_op_name[] = { + REQ_OP_NAME(READ), + REQ_OP_NAME(WRITE), + REQ_OP_NAME(FLUSH), + REQ_OP_NAME(DISCARD), + REQ_OP_NAME(WRITE_SAME), +}; +#define SINGLE_OP_NAME_SIZE 16 +#define MAX_OP_NAME_SIZE ((SINGLE_OP_NAME_SIZE + 1) * 5) + +static const char *const blk_mq_rq_state_name_array[] = { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0) + [REQ_ATOM_COMPLETE] = "complete", + [REQ_ATOM_STARTED] = "in_flight", +#else + [MQ_RQ_IDLE] = "idle", + [MQ_RQ_IN_FLIGHT] = "in_flight", + [MQ_RQ_COMPLETE] = "complete", +#endif +}; + +extern fn_get_files_struct sym_get_files_struct; +extern fn_put_files_struct sym_put_files_struct; +extern int get_bio_file_info(void); +extern void get_scsi_info(struct scsi_info *scsi_i, struct request *rq); + +static char *get_disk_name(struct gendisk *hd, int partno, char *buf) +{ + if (!partno) + snprintf(buf, BDEVNAME_SIZE, "%s", hd->disk_name); + else if (isdigit(hd->disk_name[strlen(hd->disk_name)-1])) + snprintf(buf, BDEVNAME_SIZE, "%sp%d", hd->disk_name, partno); + else + snprintf(buf, BDEVNAME_SIZE, "%s%d", hd->disk_name, partno); + return buf; +} + +static void blk_rq_op_name(int op_flags, char *op_buf, int buf_len) +{ + int i = 0; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0) + for (; i < (sizeof(g_op_name) / sizeof(g_op_name[0])); i++) { + if (op_flags == g_op_name[i].op) { + strcat(op_buf, g_op_name[i].op_str); + return; + } + } +#else + int len; + for (; i < (sizeof(g_op_name) / sizeof(g_op_name[0])); i++) { + if (op_flags & g_op_name[i].op) { + if ((len = strlen(op_buf)) >= buf_len) + return; + if (len) { + strncat(op_buf, "|", min((strlen("|") + 1), + (buf_len - len))); + op_buf[buf_len - 1] = '\0'; + if ((len = strlen(op_buf)) >= buf_len) + return; + } + strncat(op_buf, g_op_name[i].op_str, + min((strlen(g_op_name[i].op_str) + 1), + (buf_len - len))); + op_buf[buf_len - 1] = '\0'; + } + } +#endif +} + +static const char *blk_mq_rq_state_name(unsigned int rq_state) +{ + if (WARN_ON_ONCE(rq_state >= + ARRAY_SIZE(blk_mq_rq_state_name_array))) + return "(?)"; + return blk_mq_rq_state_name_array[rq_state]; +} + +static char *__dentry_name(struct dentry *dentry, char *name) +{ + char *p = dentry_path_raw(dentry, name, PATH_MAX); + + if (IS_ERR(p)) { + __putname(name); + return NULL; + } + + if (p + strlen(p) + 1 != name + PATH_MAX) { + __putname(name); + return NULL; + } + + if (p > name) + strcpy(name, p); + + return name; +} + +static char *dentry_name(struct dentry *dentry) +{ + char *name = __getname(); + if (!name) + return NULL; + + return __dentry_name(dentry, name); +} + +static char *inode_name(struct inode *ino) +{ + struct dentry *dentry; + char *name; + + dentry = d_find_alias(ino); + if (!dentry) + return NULL; + + name = dentry_name(dentry); + dput(dentry); + return name; +} + +static int is_task_open_file(struct task_struct *p, struct inode *ino) +{ + struct files_struct *files; + struct file *file; + struct fdtable *fdt; + unsigned int fd; + + files = sym_get_files_struct(p); + if (files) { + rcu_read_lock(); + fdt = files_fdtable(files); + fd = find_first_bit(fdt->open_fds, fdt->max_fds); + while (fd < fdt->max_fds) { + file = fcheck_files(files, fd); + if (file && (file_inode(file) == ino)) { + rcu_read_unlock(); + sym_put_files_struct(files); + return 1; + } + fd = find_next_bit(fdt->open_fds, fdt->max_fds, fd + 1); + } + rcu_read_unlock(); + sym_put_files_struct(files); + } + return 0; +} + +static void get_task_info_lsof(struct inode *ino, unsigned int *pid, + char *comm) +{ + struct task_struct *p; + + rcu_read_lock(); + for_each_process(p) { + if (p->flags & PF_KTHREAD) + continue; + rcu_read_unlock(); + get_task_struct(p); + if (is_task_open_file(p, ino)) { + *pid = p->pid; + memcpy(comm, p->comm, sizeof(p->comm)); + put_task_struct(p); + return; + } + put_task_struct(p); + + cond_resched(); + rcu_read_lock(); + } + rcu_read_unlock(); +} + +static int get_inode_filename(struct inode *ino, char *name_buf, + int len) +{ + char *name; + + if (!ino->i_ino) + return -1; + + name = inode_name(ino); + if (name) { + if (strlen(name) + 1 <= len) + strlcpy(name_buf, name, strlen(name) + 1); + else { + strlcpy(name_buf, "...", 4); + strlcpy(name_buf + 3, + name + (strlen(name) + 1 - (len - 3)), + (len - 3)); + } + __putname(name); + return 0; + } + return -1; +} + +static void get_bio_info(struct bio_info *bio_i, struct bio *bio) +{ + struct bio_vec *bvec; + int i; + + if (!bio) + return; + + bio_i->bio_addr = (unsigned long)bio; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0) + bio_i->sector = bio->bi_iter.bi_sector; + bio_i->size = bio->bi_iter.bi_size; +#else + bio_i->sector = bio->bi_sector; + bio_i->size = bio->bi_size; +#endif + if (get_bio_file_info()) { + bio_for_each_segment_all(bvec, bio, i) { + struct page *page = bvec->bv_page; + + if (!page) + continue; + if (page->mapping && page->mapping->host) { + if (get_inode_filename(page->mapping->host, bio_i->filename, + sizeof(bio_i->filename))) { + continue; + } + + if (sym_get_files_struct && sym_put_files_struct) + get_task_info_lsof(page->mapping->host, &bio_i->pid, + bio_i->comm); + break; + } + } + } +} + +static void get_rq_info(struct rq_hang_info *rq_hi, struct request *rq) +{ + char op_buf[MAX_OP_NAME_SIZE]; + int op; + + rq_hi->data_len = rq->__data_len; + rq_hi->sector = rq->__sector; + strcpy(op_buf, ""); + //rq_hi->cmd_flags = rq->cmd_flags; + //rq_hi->errors = rq->errors; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0) + //op = rq->cmd_flags & REQ_OP_MASK; + op = req_op(rq); + blk_rq_op_name(op, op_buf, sizeof(op_buf)); +#else + blk_rq_op_name(rq->cmd_flags, op_buf, sizeof(op_buf)); +#endif + strncpy(rq_hi->op, op_buf, min(strlen(op_buf), sizeof(rq_hi->op) - 1)); +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0) + strcpy(rq_hi->state, (blk_mq_rq_state_name((test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags) ? + REQ_ATOM_COMPLETE : REQ_ATOM_STARTED)))); +#else + strcpy(rq_hi->state, blk_mq_rq_state_name(READ_ONCE(rq->state))); +#endif + rq_hi->tag = rq->tag; + rq_hi->internal_tag = get_rq_internal_tag(rq); + if (rq->mq_ctx) + rq_hi->cpu = rq->mq_ctx->cpu; + else + rq_hi->cpu = rq->cpu; + rq_hi->io_start_ns = rq->start_time_ns; + rq_hi->io_issue_driver_ns = get_issue_driver_ns(rq); + if (rq->rq_disk) + get_disk_name(rq->rq_disk, rq->part ? rq->part->partno : 0, + rq_hi->diskname); + get_bio_info(&rq_hi->first_bio, rq->bio); +} + +int fill_hang_info_from_rq(struct rq_hang_info *rq_hi, + struct request *rq, int disk_type) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 18, 0) + if (!rq || !test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) + return -1; +#else + if (!rq || !refcount_read(&rq->ref)) + return -1; +#endif + get_rq_info(rq_hi, rq); + if (disk_type == DISK_VIRTIO_BLK) + get_vq_info(&rq_hi->vq, rq); + else if (disk_type == DISK_NVME) + get_nvme_info(&rq_hi->nvme, rq); + else if (disk_type == DISK_SCSI) + get_scsi_info(&rq_hi->scsi, rq); + return 0; +} + diff --git a/source/lib/internal/kernel_module/modules/iosdiag/scsi.c b/source/lib/internal/kernel_module/modules/iosdiag/scsi.c new file mode 100644 index 00000000..8c67174b --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/scsi.c @@ -0,0 +1,28 @@ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" + +void get_scsi_info(struct scsi_info *scsi_i, struct request *rq) +{ + struct scsi_cmnd *cmd; + + if (rq->q->mq_ops) { + scsi_i->is_mq = 1; + cmd = blk_mq_rq_to_pdu(rq); + } else + cmd = rq->special; + + if (!cmd) + return; + scsi_i->done_hander_defined = cmd->scsi_done ? 1 : 0; +} diff --git a/source/lib/internal/kernel_module/modules/iosdiag/virtio_blk.c b/source/lib/internal/kernel_module/modules/iosdiag/virtio_blk.c new file mode 100644 index 00000000..0e3dc830 --- /dev/null +++ b/source/lib/internal/kernel_module/modules/iosdiag/virtio_blk.c @@ -0,0 +1,132 @@ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct virtqueue *get_virtqueue_by_rq(struct request *rq) +{ + struct virtio_blk *vblk; + int qid; + struct blk_mq_hw_ctx *hctx; + + if (!rq) + return NULL; + + hctx = blk_mq_get_hctx_byrq(rq); + if (!hctx) + return NULL; + qid = hctx->queue_num; + vblk = hctx->queue->queuedata; + if (qid >= vblk->num_vqs) + return NULL; + return vblk->vqs[qid].vq; +} + +static struct vring *get_vring_by_vq(struct virtqueue *vq) +{ + return &to_vvq(vq)->vring; +} + +static int get_vq_id(struct request *rq) +{ + struct blk_mq_hw_ctx *hctx; + + if (!rq) + return -1; + + hctx = blk_mq_get_hctx_byrq(rq); + if (!hctx) + return -1; + return hctx->queue_num; +} + +static int get_rq_avail_idx(struct request *rq) +{ + int i; + struct vring *vring; + struct virtqueue *vq; + u16 last_used_idx; + u16 current_avail_idx; + int head; + void *data; + + if (!(vq = get_virtqueue_by_rq(rq))) + return -1; + vring = get_vring_by_vq(vq); + current_avail_idx = vring->avail->idx; + last_used_idx = to_vvq(vq)->last_used_idx; + while (last_used_idx <= current_avail_idx) { + i = last_used_idx & (vring->num - 1); + head = virtio16_to_cpu(vq->vdev, vring->avail->ring[i]); + if (head < vring->num) + if (desc_state_data_to_req(vq, head) == rq) + return last_used_idx; + else + return -1; + last_used_idx++; + } + return -1; +} + +static int get_rq_used_idx(struct request *rq) +{ + int i; + struct vring *vring; + struct virtqueue *vq; + u16 last_used_idx; + u16 used_idx; + int head; + void *data; + + if (!(vq = get_virtqueue_by_rq(rq))) + return -1; + vring = get_vring_by_vq(vq); + used_idx = virtio16_to_cpu(vq->vdev, vring->used->idx); + last_used_idx = to_vvq(vq)->last_used_idx; + while (last_used_idx < used_idx) { + i = last_used_idx & (vring->num - 1); + head = virtio32_to_cpu(vq->vdev, vring->used->ring[i].id); + if (head < vring->num) + if (desc_state_data_to_req(vq, head) == rq) + return last_used_idx; + else + return -1; + last_used_idx++; + } + return -1; +} + +void get_vq_info(struct vq_info *vq_i, struct request *rq) +{ + struct vring *vring; + struct virtqueue *vq; + + if (!(vq = get_virtqueue_by_rq(rq))) + return; + vring = get_vring_by_vq(vq); + + vq_i->qid = get_vq_id(rq); + vq_i->vring_num = vring->num; + vq_i->event = to_vvq(vq)->event ? 1 : 0; + vq_i->last_used_idx = to_vvq(vq)->last_used_idx; + vq_i->used_idx = vring->used->idx; + vq_i->used_ring_flags = vring->used->flags; + if (vq_i->event == 1) + vq_i->last_avail_idx = + *(__virtio16 *)&vring->used->ring[vring->num]; + else + vq_i->last_avail_idx = -1; + vq_i->avail_idx = vring->avail->idx; + vq_i->avail_ring_flags = vring->avail->flags; + vq_i->last_kick_avail_idx = vq_i->avail_idx - to_vvq(vq)->num_added; + vq_i->rq_avail_idx = get_rq_avail_idx(rq); + vq_i->rq_used_idx = get_rq_used_idx(rq); +} + diff --git a/source/lib/internal/kernel_module/sysak_mods.c b/source/lib/internal/kernel_module/sysak_mods.c index 11f99605..de2d940a 100644 --- a/source/lib/internal/kernel_module/sysak_mods.c +++ b/source/lib/internal/kernel_module/sysak_mods.c @@ -65,6 +65,14 @@ int __attribute__((weak)) loadtask_exit(void) { return 0; } +int __attribute__((weak)) disk_hang_init(void) +{ + return 0; +} +int __attribute__((weak)) disk_hang_exit(void) +{ + return 0; +} struct sysak_module sysak_modules[] = { { "trace_sig", trace_sig_init, trace_sig_exit}, @@ -74,6 +82,7 @@ struct sysak_module sysak_modules[] = { { "schedtrace", schedtrace_init, schedtrace_exit}, { "mmap_trace", mmaptrace_init, mmaptrace_exit}, { "loadtask", loadtask_init, loadtask_exit}, + { "iosdiag", disk_hang_init, disk_hang_exit}, }; const int sysk_module_num = sizeof(sysak_modules) / sizeof(struct sysak_module); diff --git a/source/tools/detect/iosdiag/data_analysis/iosdiag_data_analysis.py b/source/tools/detect/iosdiag/data_analysis/iosdiag_data_analysis.py index 4a1410df..b007456f 100755 --- a/source/tools/detect/iosdiag/data_analysis/iosdiag_data_analysis.py +++ b/source/tools/detect/iosdiag/data_analysis/iosdiag_data_analysis.py @@ -8,7 +8,7 @@ import argparse import re if os.geteuid() != 0: - print "This program must be run as root. Aborting." + print("This program must be run as root. Aborting.") sys.exit(0) def execCmd(cmd): @@ -71,7 +71,7 @@ class latencyAnalysis: self.totalDelayDicts.setdefault(disk, 0) self.totalIosDicts.setdefault(disk, 0) self.diskIdxDicts.setdefault(disk, self.totalDiskCnt) - self.totalDiskCnt += 1; + self.totalDiskCnt += 1 def processLatencyDelays(self, sDict): diskIdxDicts = self.diskIdxDicts @@ -100,7 +100,7 @@ class latencyAnalysis: disk = sDict['diskname'] diskIdx = diskIdxDicts[disk] del sDict['diskname'] - listAbnormal=[i for i in sDict['abnormal'].split(' ') if i != '']; + listAbnormal=[i for i in sDict['abnormal'].split(' ') if i != ''] msDelay=int(listAbnormal[-2].strip('(').split(':')[0]) / 1000.000 msTotalDelay=int(listAbnormal[-2].strip('(').split(':')[1]) / 1000.000 sDict['abnormal']=listAbnormal[0]+' '+listAbnormal[1]+" ("+str(msDelay)+":"+str(msTotalDelay)+" ms)" @@ -162,14 +162,18 @@ class latencyAnalysis: end = totalIos if totalIos < 10 else 10 print("The first %d IOs with the largest delay, more details:" % end) - print("time".ljust(26)+"comm".ljust(20)+"pid".ljust(10)+"iotype".ljust(8)+\ - "datalen".ljust(16)+"abnormal(delay:totaldelay)".ljust(40)) + print("%-26s%-20s%-10s%-8s%-16s%s" % \ + ('time', 'comm', 'pid', 'iotype', 'datalen', 'abnormal(delay:totaldelay)')) for i in range(0,end): eDict=summaryDicts['summary'][diskIdx]['slow ios'][i] - print(str(eDict["time"]).ljust(26)+eDict["comm"].ljust(20)+\ - str(eDict["pid"]).ljust(10)+eDict["iotype"].ljust(8)+\ - str(eDict["datalen"]).ljust(16)+eDict["abnormal"].ljust(40)) + print("%-26s%-20s%-10s%-12s%-12s%s" % \ + (str(eDict["time"]), \ + eDict["comm"], \ + str(eDict["pid"]),\ + eDict["iotype"],\ + str(eDict["datalen"]),\ + eDict["abnormal"])) def latencyDataAnalysis(resultSeqFile, threshold): analysis = latencyAnalysis() @@ -200,24 +204,187 @@ def latencyDataAnalysis(resultSeqFile, threshold): analysis.latencyPrint(int(threshold)) print("more details see %s*" % resultSeqFile.strip('.seq')) +class hangAnalysis(): + def __init__(self): + self.statDicts = {} + self.summaryDicts = {} + self.detailDicts = {} + self.totalIosDicts = {} + self.diskIdxDicts = {} + self.totalDiskCnt = 0 + self.statJsonStr = \ + '{ \ + "diskname":"",\ + "hung ios":[ \ + {"component":"os","count":0,"percent":"","max":0,"min":1000000000,"avg":0},\ + {"component":"disk","count":0,"percent":"","max":0,"min":1000000000,"avg":0}] \ + }' + newStatDict = json.loads("["+self.statJsonStr + "]", object_pairs_hook=OrderedDict) + self.statDicts.setdefault('summary', newStatDict) + self.entryDictJsonStr = \ + '{ \ + "diskname":"",\ + "hung ios":[] \ + }' + newSummaryDict = json.loads("["+self.entryDictJsonStr + "]", object_pairs_hook=OrderedDict) + self.summaryDicts.setdefault('summary', newSummaryDict) + newDetailDicts = json.loads("["+self.entryDictJsonStr + "]", object_pairs_hook=OrderedDict) + self.detailDicts.setdefault('summary', newDetailDicts) + + def __newDiskDict(self, disk): + if self.totalDiskCnt != 0: + newStatDicts = json.loads(self.statJsonStr, object_pairs_hook=OrderedDict) + self.statDicts['summary'].append(newStatDicts) + newSummaryDict = json.loads(self.entryDictJsonStr, object_pairs_hook=OrderedDict) + self.summaryDicts['summary'].append(newSummaryDict) + newDetailDicts = json.loads(self.entryDictJsonStr, object_pairs_hook=OrderedDict) + self.detailDicts['summary'].append(newDetailDicts) + self.statDicts['summary'][self.totalDiskCnt]['diskname'] = disk + self.summaryDicts['summary'][self.totalDiskCnt]['diskname'] = disk + self.detailDicts['summary'][self.totalDiskCnt]['diskname'] = disk + self.totalIosDicts.setdefault(disk, 0) + self.diskIdxDicts.setdefault(disk, self.totalDiskCnt) + self.totalDiskCnt += 1 + + def processOneHangSeq(self, sDict): + statDicts = self.statDicts + totalIosDicts = self.totalIosDicts + diskIdxDicts = self.diskIdxDicts + summaryDicts = self.summaryDicts + detailDicts = self.detailDicts + components = ['OS', 'Disk'] + + disk = sDict['diskname'] + del sDict['diskname'] + if disk not in totalIosDicts.keys(): + self.__newDiskDict(disk) + + diskIdx = diskIdxDicts[disk] + if "abnormal" in str(sDict): + totalIosDicts[disk] += 1 + statDict = statDicts['summary'][diskIdx] + abnormalList = sDict['abnormal'].split() + hungComponentIdx = components.index(abnormalList[0].split('(')[0]) + hungDelay = long(abnormalList[-2]) + statDict['hung ios'][hungComponentIdx]['count'] += 1 + maxHungDelay = statDict['hung ios'][hungComponentIdx]['max'] + minHungDelay = statDict['hung ios'][hungComponentIdx]['min'] + if hungDelay > maxHungDelay: + statDict['hung ios'][hungComponentIdx]['max'] = hungDelay + if hungDelay < minHungDelay: + statDict['hung ios'][hungComponentIdx]['min'] = hungDelay + statDict['hung ios'][hungComponentIdx]['avg'] += hungDelay + abnormalList[-2] = str(round(hungDelay / 1000000.000, 3)) + abnormalList[-1] = 's' + sDict['abnormal'] = ''.join(str(e)+' ' for e in abnormalList).strip() + summaryDicts['summary'][diskIdx]['hung ios'].append(sDict) + else: + detailDicts['summary'][diskIdx]['hung ios'].append(sDict) + + def hangStatCalculate(self): + statDicts = self.statDicts + totalIosDicts = self.totalIosDicts + diskIdxDicts = self.diskIdxDicts + components = {'OS':0, 'Disk':1} + + for disk, diskIdx in diskIdxDicts.items(): + for component, idx in components.items(): + if statDicts['summary'][diskIdx]['hung ios'][idx]['count'] != 0: + avgDelay = statDicts['summary'][diskIdx]['hung ios'][idx]['avg'] + avgDelay /= (statDicts['summary'][diskIdx]['hung ios'][idx]['count']) + statDicts['summary'][diskIdx]['hung ios'][idx]['avg'] = avgDelay + percent = statDicts['summary'][diskIdx]['hung ios'][idx]['count'] * 100.0 / totalIosDicts[disk] + else: + statDicts['summary'][diskIdx]['hung ios'][idx]['min'] = 0 + percent = 0 + statDicts['summary'][diskIdx]['hung ios'][idx]['percent'] = str(round(percent, 3)) + "%" + statDicts['summary'][diskIdx]['hung ios'][idx]['component'] = component + + def hangPrint(self): + diskIdxDicts = self.diskIdxDicts + totalIosDicts = self.totalIosDicts + summaryDicts = self.summaryDicts + statDicts = self.statDicts + components = {'OS':0, 'Disk':1} + + for disk, diskIdx in diskIdxDicts.items(): + totalIos = totalIosDicts[disk] + summaryDicts['summary'][diskIdx]['hung ios']=\ + sorted(summaryDicts['summary'][diskIdx]['hung ios'],\ + key=lambda e:float(e['abnormal'].split()[-2]),\ + reverse=True) + print("\n%d IOs hung of disk %s, IO hung distribution:" %(totalIos, disk)) + for component,idx in components.items(): + percent = statDicts['summary'][diskIdx]['hung ios'][idx]['percent'] + print("%-12s delay: %s" %(component, percent)) + + end = totalIos if totalIos < 10 else 10 + print("The first %d IOs with the largest delay, more details:" % end) + print("%-26s%-20s%-10s%-12s%-12s%-12s%-50s%s" % \ + ('time', 'comm', 'pid', 'iotype', 'sector', 'datalen', 'abnormal', 'file')) + + for i in range(0,end): + eDict=summaryDicts['summary'][diskIdx]['hung ios'][i] + print("%-26s%-20s%-10s%-12s%-12s%-12s%-50s%s" % \ + (str(eDict["time"]), \ + eDict["comm"] if len(eDict["comm"]) else '-', \ + str(eDict["pid"]) if eDict["pid"] != -1 else '-',\ + eDict["iotype"],\ + str(eDict["sector"]),\ + str(eDict["datalen"]),\ + eDict["abnormal"], \ + eDict["file"] if len(eDict["file"]) else '-')) + +def hangDataAnalysis(resultSeqFile, threshold): + analysis = hangAnalysis() + f = open(resultSeqFile) + for line in f.readlines(): + try: + sDict = json.loads(line, object_pairs_hook=OrderedDict) + except ValueError: + continue + analysis.processOneHangSeq(sDict) + f.close() + + if analysis.totalDiskCnt == 0: + print("\nnot IO hang, everything is ok !^o^ ~") + return + analysis.hangStatCalculate() + + f = open(resultSeqFile.strip('.seq')+".stat", 'w+') + f.write(json.dumps(analysis.statDicts)) + f.close() + + f = open(resultSeqFile.strip('.seq'), 'w+') + f.write(json.dumps(analysis.summaryDicts)) + f.close() + + f = open(resultSeqFile, 'w+') + f.write(json.dumps(analysis.detailDicts)) + f.close() + analysis.hangPrint() + print("more details see %s*" % resultSeqFile.strip('.seq')) + def main(): examples = """e.g. ./iosdiag_data_analysis.py -L -s -t 1000 -f ./result.log.seq //Statistic IO delay diagnosis results - ./iosdiag_data_analysis.py -L -g -t 1000 -f ./result.log.seq //Display IO delay diagnostic results graphically + ./iosdiag_data_analysis.py -H -s -t 2000 -f ./result.log.seq //Statistic IO hang diagnosis results """ parser = argparse.ArgumentParser( description="Analyze IO diagnostic data.", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=examples) parser.add_argument('-L','--latency', action='store_true', help='Analyze IO delay diagnostic data.') + parser.add_argument('-H','--hangdetect', action='store_true', help='Analyze IO hang diagnostic data.') parser.add_argument('-s','--stat', action='store_true', help='Statistic IO diagnosis results.') - parser.add_argument('-g','--graph', action='store_true', help='Display IO diagnostic results graphically.') parser.add_argument('-t','--threshold', help='Specifies the threshold for the exception io.') parser.add_argument('-f','--file', help='Specify the target data file to analyze.') args = parser.parse_args() if args.latency: latencyDataAnalysis(args.file, 1000 if args.threshold == None else args.threshold) + if args.hangdetect: + hangDataAnalysis(args.file, 5000 if args.threshold == None else args.threshold) if __name__ == "__main__": main() diff --git a/source/tools/detect/iosdiag/entry/iosdiag.sh b/source/tools/detect/iosdiag/entry/iosdiag.sh index 10e442c5..6d69b71a 100755 --- a/source/tools/detect/iosdiag/entry/iosdiag.sh +++ b/source/tools/detect/iosdiag/entry/iosdiag.sh @@ -15,37 +15,87 @@ fi TOOLS_PATH=$WORK_PATH/tools/`uname -r` LIB_PATH=$WORK_PATH/lib/`uname -r` latency_bin=$WORK_PATH/tools/latency +hangdetect_bin=$TOOLS_PATH/hangdetect data_analysis=$WORK_PATH/tools/iosdiag_data_analysis -iosdiag_dir="/var/log/sysak/iosdiag" -logfile="$iosdiag_dir/$1/result.log.seq" -logfile_arg="-f $logfile" +logfile="/var/log/sysak/iosdiag/$1/result.log.seq" +threshold_arg="-t 1000" function usage() { - echo "Usage: sysak iosdiag [options] subcmd [cmdargs]]" + echo "Usage: sysak iosdiag [options] subcmd [cmdargs]" echo " subcmd:" echo " latency, io latency diagnosis" + echo " hangdetect, io hang diagnosis" echo " cmdargs:" echo " -h, help info" echo " options:" echo " -u url, transfer datafile to remote url" - echo " -s latency|[..], stop diagnosis" - #echo " -f logfile, output log file" -} - -function list() { - ls $TOOLS_PATH + echo " -s latency|[hangdetect], stop diagnosis" } upload_data() { - datapath=$iosdiag_dir/$1 + datapath=$(dirname $logfile) cd $datapath - tar -zcf iosdiag_$1.tar.gz ./* + tar -zcf iosdiag_$1.tar.gz ./result.log* curl -i -q -X PUT -T iosdiag_$1.tar.gz $url rm -f iosdiag_$1.tar.gz } datafile_analysis() { - python $data_analysis --$1 --stat --file $logfile $threshold_arg + if [ -e "$logfile" ] + then + python $data_analysis --$1 -s -f $logfile $threshold_arg + fi +} + +hang_mod_depend() +{ + res=`lsmod | grep sysak` + if [ -z "$res" ]; then + insmod $LIB_PATH/sysak.ko + if [ $? -ne 0 ]; then + echo "insmod ko failed, please check the ko files." + exit $? + fi + fi +} + +enable_hangdetect() { + if [ ! -e "$hangdetect_bin" ]; then + echo "$hangdetect_bin not found" + echo "iosdiag hangdetect not support '$(uname -r)', please report to the developer" + exit -1 + fi + { + flock -n 3 + [ $? -eq 1 ] && { echo "another hangdetect is running."; exit -1; } + trap disable_hangdetect SIGINT SIGTERM SIGQUIT + #mkdir -p `dirname $datafile` + hang_mod_depend + chmod +x $hangdetect_bin + rm -f $(dirname $logfile)/result.log* + $hangdetect_bin $* + wait $! + disable_hangdetect + } 3<> /tmp/hangdetect.lock +} + +disable_hangdetect() { + pid=`ps -ef | grep "\$hangdetect_bin" | awk '{print $2}'` + if [ "$pid" != "" ] + then + kill -9 $pid 2>/dev/null + fi + + res=`lsmod | grep sysak` + if [ ! -z "$res" ]; then + rmmod sysak + fi + + datafile_analysis hangdetect + if [ -n "$url" ]; then + upload_data hangdetect + fi + exit 0 } enable_latency() { @@ -54,17 +104,15 @@ enable_latency() { echo "iosdiag latency not support '$(uname -r)', please report to the developer" exit -1 fi - threshold=$(echo "$*"|awk -F "-t" '{print $2}'|awk '{print $1}') - [ "$threshold" != "" ] && { threshold_arg="-t $threshold"; } { flock -n 3 [ $? -eq 1 ] && { echo "another latency is running."; exit -1; } trap disable_latency SIGINT SIGTERM SIGQUIT #mkdir -p `dirname $datafile` chmod +x $latency_bin - rm $logfile_arg + rm -f $(dirname $logfile)/result.log* $SYSAK_WORK_PATH/../sysak btf - $latency_bin $logfile_arg $* & + $latency_bin $* & wait $! disable_latency } 3<> /tmp/latency.lock @@ -91,6 +139,10 @@ disable_latency() { # output format jason # -d/--disable function execute() { + threshold=$(echo "$*"|awk -F "-t" '{print $2}'|awk '{print $1}') + [ "$threshold" != "" ] && { threshold_arg="-t $threshold"; } + logd=$(echo "$*"|awk -F "-f" '{print $2}'|awk '{print $1}') + [ "$logd" != "" ] && { logfile=$logd/result.log.seq; } #echo cmd:$1 ${*:2} enable_$1 ${*:2} } @@ -119,6 +171,6 @@ fi subcmd=${@:$OPTIND:1} subargs=${*:$OPTIND+1}; -[ "$subcmd" != "latency" ] && { echo "not support subcmd $subcmd!!!"; usage; exit -1; } +[ "$subcmd" != "latency" -a "$subcmd" != "hangdetect" ] && { echo "not support subcmd $subcmd!!!"; usage; exit -1; } execute $subcmd $subargs diff --git a/source/tools/detect/iosdiag/hangdetect/Makefile b/source/tools/detect/iosdiag/hangdetect/Makefile new file mode 100644 index 00000000..0858d781 --- /dev/null +++ b/source/tools/detect/iosdiag/hangdetect/Makefile @@ -0,0 +1,11 @@ +target = hangdetect +ifneq ($(wildcard $(SRC)/lib/internal/kernel_module/modules/iosdiag/include/$(KERNEL_VERSION)),) +mods := main.o base_info.o json_format.o +KERNEL_DEPEND=Y + +LDFLAGS += -lrt +include $(SRC)/mk/csrc.mk +else +all: +endif + diff --git a/source/tools/detect/iosdiag/hangdetect/base_info.c b/source/tools/detect/iosdiag/hangdetect/base_info.c new file mode 100644 index 00000000..79341d85 --- /dev/null +++ b/source/tools/detect/iosdiag/hangdetect/base_info.c @@ -0,0 +1,159 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "base_info.h" + +#define MAX_BDI_CNT 128 + +static struct base_info g_bi; +static int g_bdi_cnt; +static int g_bni_cnt; +static int g_current_bdi_idx; + +static int disk_info_init(struct base_disk_info *bdi, int max_bdi_cnt) +{ + FILE *fp; + char buf[256] = {0}; + char *cmd = "lsblk -d -P -o NAME,MAJ:MIN"; + + if ((fp = popen(cmd, "r")) == NULL) { + fprintf(stderr, "exec \'%s\' fail\n", cmd); + return -1; + } + while (fgets(buf, sizeof(buf) - 1, fp)) { + sscanf(buf, "NAME=\"%[^\"]\" MAJ:MIN=\"%d:%d\"", + bdi[g_bdi_cnt].diskname, + &bdi[g_bdi_cnt].major, + &bdi[g_bdi_cnt].minor); + g_bdi_cnt++; + if (g_bdi_cnt >= max_bdi_cnt) + break; + } + pclose(fp); + return 0; +} + +static int mnt_info_init(struct base_mnt_info *bni, int max_bni_cnt) +{ + FILE *fp; + char buf[256] = {0}; + char *cmd = "lsblk -P -o NAME,MOUNTPOINT | grep \"/\""; + + if ((fp = popen(cmd, "r")) == NULL) { + fprintf(stderr, "exec \'%s\' fail\n", cmd); + return -1; + } + while (fgets(buf, sizeof(buf) - 1, fp)) { + sscanf(buf, "NAME=\"%[^\"]\" MOUNTPOINT=\"%[^\"]\"", + bni[g_bni_cnt].diskname, + bni[g_bni_cnt].mnt_dir); + g_bni_cnt++; + if (g_bni_cnt >= max_bni_cnt) + break; + } + pclose(fp); + return 0; +} + +static int get_bdi_idx_by_name(char *name) +{ + int i; + + if (!g_bi.bdi) + return -1; + + for (i = 0; i < MAX_BDI_CNT && g_bi.bdi[i].major; i++) + if (!strncmp(g_bi.bdi[i].diskname, name, + strlen(g_bi.bdi[i].diskname))) + return i; + return -1; +} + +int set_current_bdi_idx(int idx) +{ + if (idx >= g_bdi_cnt) + return -1; + g_current_bdi_idx = idx; + return 0; +} + +int get_current_bdi_idx(void) +{ + return g_current_bdi_idx; +} + +int get_bdi_cnt(void) +{ + return g_bdi_cnt; +} + +struct base_info *get_base_info_ptr(void) +{ + return &g_bi; +} + +struct base_disk_info *get_current_bdi(void) +{ + return &g_bi.bdi[g_current_bdi_idx]; +} + +char *get_bdi_mnt_dir(char *name) +{ + int i = 0; + + if (!g_bi.bni) + return NULL; + + for (; i < g_bni_cnt; i++) + if (!strcmp(g_bi.bni[i].diskname, name)) + return g_bi.bni[i].mnt_dir; + return NULL; +} + +int base_info_init(char *diskname) +{ + int major, minor; + int current_bdi_idx; + int size; + + size = MAX_BDI_CNT * sizeof(struct base_disk_info); + g_bi.bdi = (struct base_disk_info *)malloc(size); + if (!g_bi.bdi) + return -ENOMEM; + memset(g_bi.bdi, 0x0, size); + disk_info_init(g_bi.bdi, MAX_BDI_CNT); + + size = MAX_BDI_CNT * sizeof(struct base_mnt_info); + g_bi.bni = (struct base_mnt_info *)malloc(size); + if (!g_bi.bni) + return -ENOMEM; + memset(g_bi.bni, 0x0, size); + mnt_info_init(g_bi.bni, MAX_BDI_CNT); + + current_bdi_idx = get_bdi_idx_by_name(diskname); + set_current_bdi_idx(current_bdi_idx); + return 0; +} + +void base_info_exit(void) +{ + if (g_bi.bdi) { + free(g_bi.bdi); + g_bi.bdi = NULL; + } + if (g_bi.bni) { + free(g_bi.bni); + g_bi.bni = NULL; + } +} + diff --git a/source/tools/detect/iosdiag/hangdetect/base_info.h b/source/tools/detect/iosdiag/hangdetect/base_info.h new file mode 100644 index 00000000..4ffc328c --- /dev/null +++ b/source/tools/detect/iosdiag/hangdetect/base_info.h @@ -0,0 +1,49 @@ +#ifndef __BASE_INFO_H +#define __BASE_INFO_H + +struct base_disk_info { + int major; + int minor; + char diskname[128]; +}; + +struct base_mnt_info { + char mnt_dir[128]; + char diskname[128]; +}; + +struct base_info { + char *check_time_date; + char *comm; + char *file; + struct base_disk_info *bdi; + struct base_mnt_info *bni; + int pid; +}; + +int set_current_bdi_idx(int idx); +int get_current_bdi_idx(void); +int get_bdi_cnt(void); +char *get_bdi_mnt_dir(char *diskname); +struct base_info *get_base_info_ptr(void); +struct base_disk_info *get_current_bdi(void); +int base_info_init(char *diskname); +void base_info_exit(void); + +#define get_bdi(name) get_current_bdi()->##name +#define get_base_info(name) get_base_info_ptr()->##name +#define set_base_info(name, value) (get_base_info_ptr()->##name = value) + +#define get_bdi_major() get_current_bdi()->major +#define get_bdi_minor() get_current_bdi()->minor +#define get_bdi_diskname() get_current_bdi()->diskname +#define get_base_info_pid() get_base_info_ptr()->pid +#define get_base_info_comm() get_base_info_ptr()->comm +#define get_base_info_file() get_base_info_ptr()->file +#define get_base_info_check_time_date() get_base_info_ptr()->check_time_date +#define set_base_info_pid(v) (get_base_info_ptr()->pid = v) +#define set_base_info_comm(v) (get_base_info_ptr()->comm = v) +#define set_base_info_file(v) (get_base_info_ptr()->file = v) +#define set_base_info_check_time_date(v) (get_base_info_ptr()->check_time_date = v) +#endif + diff --git a/source/tools/detect/iosdiag/hangdetect/iosdiag.h b/source/tools/detect/iosdiag/hangdetect/iosdiag.h new file mode 100644 index 00000000..fd14f2e8 --- /dev/null +++ b/source/tools/detect/iosdiag/hangdetect/iosdiag.h @@ -0,0 +1,82 @@ +#ifndef __IOSDIAG_H +#define __IOSDIAG_H + +#define MAX_STORE_RQ_CNT 128 +#define MAX_FILE_NAME_LEN 255 +#define BIO_INFO_MAX_PAGES 32 +#define MAX_REQ_BIOS 32 +#define TASK_COMM_LEN 16 + +struct vq_info { + int qid; + int vring_num; + int last_used_idx; + int used_idx; + int used_ring_flags; + int last_avail_idx; + int avail_idx; + int avail_ring_flags; + int event; + int rq_avail_idx; + int last_kick_avail_idx; + int rq_used_idx; +}; + +struct nvme_info { + int qid; + int q_depth; //sq/cq depth + int cq_head; //nvmeq->cqes[cq_head]~nvmeq->cqes[cq_end], including req->tag? + int cq_end; + int cq_rq_idx; //rq idx in cq + //int last_cq_head; //nvmeq->sq_head or nvmeq->last_cq_head + int sq_tail; //0~nvmeq->sq_cmds[idx].command_id, including req->tag? + int sq_rq_idx; //rq idx in sq + int sq_last_db; //last sq idx host kick nvme, nvmeq->q_db + unsigned long cmd_ctx; +}; + +struct scsi_info { + int done_hander_defined; + int is_mq; +}; + +/* +struct file_info { + //unsigned long page_addr; + //unsigned long i_ino; + char name[MAX_FILE_NAME_LEN]; +}; +*/ +struct bio_info { + unsigned long bio_addr; + unsigned long sector; + unsigned int size; + unsigned int pid; + char comm[TASK_COMM_LEN]; + char filename[MAX_FILE_NAME_LEN]; + //struct file_info file[BIO_INFO_MAX_PAGES]; +}; + +struct rq_hang_info { + unsigned int data_len; + unsigned long sector; + unsigned long req_addr; + unsigned long long io_start_ns; + unsigned long long io_issue_driver_ns; + unsigned long long check_hang_ns; + char op[64]; + char state[16]; + struct vq_info vq; + struct nvme_info nvme; + struct scsi_info scsi; + int tag; + int internal_tag; + int cpu; + //int errors; + //unsigned long cmd_flags; + char diskname[32]; + //struct bio_info bio[MAX_REQ_BIOS]; + struct bio_info first_bio; +}; +#endif + diff --git a/source/tools/detect/iosdiag/hangdetect/json_format.c b/source/tools/detect/iosdiag/hangdetect/json_format.c new file mode 100644 index 00000000..a5f929a9 --- /dev/null +++ b/source/tools/detect/iosdiag/hangdetect/json_format.c @@ -0,0 +1,284 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" +#include "base_info.h" +#include "json_format.h" + +enum disk_type { + DISK_VIRTIO_BLK, + DISK_SCSI, + DISK_NVME +}; + +static int get_disk_type(char *buf) +{ + if (buf[0] == 'v' && buf[1] == 'd' && (buf[2] >= 'a' && buf[2] <= 'z')) + return DISK_VIRTIO_BLK; + else if (buf[0] == 's' && buf[1] == 'd' && (buf[2] >= 'a' && buf[2] <= 'z')) + return DISK_SCSI; + else if (!strncmp(buf, "nvme", 4)) + return DISK_NVME; + return -1; +} + +static char *get_abnormal_nvme(struct nvme_info *nvme) +{ + if ((nvme->cmd_ctx & 0xFFF) == 0x310 || (nvme->cmd_ctx & 0xFFF) == 0x30C) + return "OS(Waitting for IO complete)"; + + if (nvme->cq_rq_idx != -1 && nvme->cq_rq_idx >= nvme->cq_head) + return "OS(Waitting for Irq handle)"; + + if (nvme->sq_rq_idx != -1) { + if (nvme->sq_rq_idx < nvme->sq_last_db) + return "Disk(Waitting for disk handle)"; + + if (nvme->sq_rq_idx >= nvme->sq_last_db && nvme->sq_rq_idx < nvme->sq_tail) + return "OS(Driver not issue to disk)"; + } + return "Unkown"; +} + +static char *get_abnormal_scsi(struct scsi_info *scsi) +{ + if (!scsi->done_hander_defined) + return "OS(Waitting for SCSI-driver dispatch cmd)"; + return "Disk(Waitting for disk handle)"; +} + +static char *get_abnormal_vq(struct vq_info *vq) +{ + //device exec it done + if (vq->rq_used_idx != -1 && vq->rq_used_idx < vq->used_idx) + return "OS(Waitting for Irq handle)"; + + if (vq->rq_avail_idx != -1) { + //io issue to device, but device not exec it + if (vq->rq_avail_idx < vq->last_kick_avail_idx) + if (vq->rq_avail_idx >= vq->used_idx) + //device not exec it + return "Disk(Waitting for disk handle)"; + else + if (vq->rq_avail_idx < vq->last_used_idx) + return "OS(Waitting for IO complete)"; + else + return "OS(Waitting for Irq handle)"; + else + //io don't issue to device + return "OS(Waitting for Driver issue to disk)"; + } + + if (vq->event) { + if (vq->avail_idx == vq->last_avail_idx && vq->last_avail_idx == vq->used_idx + && vq->used_idx == vq->last_used_idx) + return "OS(Waitting for IO complete)"; + + if (vq->avail_idx == vq->last_avail_idx && vq->last_avail_idx == vq->used_idx + && vq->used_idx > vq->last_used_idx) + return "OS(Waitting for Irq handle)"; + + if (vq->avail_idx == vq->last_avail_idx && vq->last_avail_idx > vq->used_idx + && vq->used_idx == vq->last_used_idx) + return "Disk(Waitting for disk handle)"; + + if (vq->avail_idx == vq->last_avail_idx && vq->last_avail_idx > vq->used_idx + && vq->used_idx > vq->last_used_idx) + return "Disk(Waitting for disk handle)"; + + if (vq->avail_idx > vq->last_avail_idx && vq->last_avail_idx > vq->used_idx + && vq->used_idx == vq->last_used_idx) + return "Disk(Waitting for disk handle)"; + + if (vq->avail_idx > vq->last_avail_idx && vq->last_avail_idx > vq->used_idx + && vq->used_idx > vq->last_used_idx) + return "Disk(Waitting for disk handle)"; + } + return "Unkown"; +} + +static char *get_abnormal(struct rq_hang_info *rq_hi) +{ + if (rq_hi->io_issue_driver_ns == 0) + return "OS(Waitting for Block issue to driver)"; + if (!strcmp(rq_hi->state, "complete")) + return "OS(Waitting for IO complete)"; + + if (get_disk_type(get_bdi_diskname()) == DISK_NVME) + return get_abnormal_nvme(&rq_hi->nvme); + else if (get_disk_type(get_bdi_diskname()) == DISK_SCSI) + return get_abnormal_scsi(&rq_hi->scsi); + else if (get_disk_type(get_bdi_diskname()) == DISK_VIRTIO_BLK) + return get_abnormal_vq(&rq_hi->vq); + else + return "Unkown"; +} + +static void summary_convert_to_json(char *dest, struct rq_hang_info *rq_hi) +{ + sprintf(dest, "{\"time\":\"%s\"," + "\"abnormal\":\"%s hang %ld us\"," + "\"diskname\":\"%s\"," + "\"iotype\":\"%s\"," + "\"sector\":%lu," + "\"datalen\":%u," + "\"iostate\":\"%s\"," + "\"cpu\":%d," + "\"comm\":\"%s\"," + "\"pid\":%d," + "\"file\":\"%s%s\"}", + get_base_info_check_time_date(), + get_abnormal(rq_hi), + (rq_hi->check_hang_ns - rq_hi->io_start_ns) / 1000, + rq_hi->diskname, + rq_hi->op, + rq_hi->sector, + rq_hi->data_len, + rq_hi->state, + rq_hi->cpu, + get_base_info_comm() ? : "", + get_base_info_pid() ? : -1, + get_base_info_file() ? get_bdi_mnt_dir(rq_hi->diskname) : "", + get_base_info_file() ? : ""); + //rq_hi->errors, + //rq_hi->cmd_flags); +} + +static void rqinfo_convert_to_json(char *dest, struct rq_hang_info *rq_hi) +{ + int real_bio_cnt = 0; + + sprintf(dest, "\"rq_info\":{" + "\"tag\":%d," + "\"internal_tag\":%d," + "\"io_start\":%llu," + "\"io_issue_driver\":%llu},", + rq_hi->tag, + rq_hi->internal_tag, + rq_hi->io_start_ns / 1000, + rq_hi->io_issue_driver_ns / 1000); + if (rq_hi->first_bio.bio_addr) { + struct bio_info *bio = &rq_hi->first_bio; + + if (strlen(bio->filename)) + set_base_info_file(bio->filename); + set_base_info_comm(bio->comm); + set_base_info_pid(bio->pid); + } +} + + +static void vqinfo_convert_to_json(char *dest, struct vq_info *vq) +{ + sprintf(dest, "\"vq_info\":{" + "\"qid\":%d," + "\"vring_num\":%d," + "\"event\":%d," + "\"last_used_idx\":%d," + "\"used_idx\":%d," + "\"last_avail_idx\":%d," + "\"avail_idx\":%d," + "\"last_kick_avail_idx\":%d," + "\"rq_avail_idx\":%d," + "\"rq_used_idx\":%d}", + vq->qid, + vq->vring_num, + vq->event, + vq->last_used_idx, + vq->used_idx, + vq->last_avail_idx, + vq->avail_idx, + vq->last_kick_avail_idx, + vq->rq_avail_idx, + vq->rq_used_idx, + vq->used_ring_flags, + vq->avail_ring_flags); +} + +static char *cmd_ctx_to_str(unsigned long cmd_ctx) +{ + static char cmd_ctx_buf[24]; + + switch (cmd_ctx & 0xFFF) { + case 0x310: + return "CMD_CTX_COMPLETED"; + case 0x30C: + return "CMD_CTX_CANCELLED"; + case 0x314: + return "CMD_CTX_INVALID"; + default: + sprintf(cmd_ctx_buf, "%#lx", cmd_ctx); + return cmd_ctx_buf; + } +} + +static void nvmeinfo_convert_to_json(char *dest, struct nvme_info *nvme) +{ + sprintf(dest, "\"nvme_info\":{" + "\"qid\":%d," + "\"q_depth\":%d," + "\"cq_head\":%d," + "\"cq_end\":%d," + "\"cq_rq_idx\":%d," + "\"sq_tail\":%d," + "\"sq_rq_idx\":%d," + "\"last_sq_tail_db\":%d," + "\"cmd_ctx\":\"%s\"}", + nvme->qid, + nvme->q_depth, + nvme->cq_head, + nvme->cq_end, + nvme->cq_rq_idx, + nvme->sq_tail, + nvme->sq_rq_idx, + nvme->sq_last_db, + cmd_ctx_to_str(nvme->cmd_ctx)); +} + +static void scsiinfo_convert_to_json(char *dest, struct scsi_info *scsi) +{ + sprintf(dest, "\"scsi_info\":{" + "\"is_mq\":\"%s\"," + "\"done_hander_defined\":\"%s\"}", + scsi->is_mq ? "ture" : "false", + scsi->done_hander_defined ? "ture" : "false"); +} + +static void detail_convert_to_json(char *dest, struct rq_hang_info *rq_hi) +{ + char *diskname = get_bdi_diskname(); + + sprintf(dest, "{\"diskname\":\"%s\"," + "\"time\":\"%s\",", + rq_hi->diskname, + get_base_info_check_time_date()); + rqinfo_convert_to_json(dest + strlen(dest), rq_hi); + if (get_disk_type(get_bdi_diskname()) == DISK_VIRTIO_BLK) + vqinfo_convert_to_json(dest + strlen(dest), &rq_hi->vq); + else if (get_disk_type(get_bdi_diskname()) == DISK_NVME) + nvmeinfo_convert_to_json(dest + strlen(dest), &rq_hi->nvme); + else if (get_disk_type(get_bdi_diskname()) == DISK_SCSI) + scsiinfo_convert_to_json(dest + strlen(dest), &rq_hi->scsi); + sprintf(dest + strlen(dest), "%s", "}"); +} + +void convert_to_json(char *dest, void *src) +{ + struct rq_hang_info *rq_hi = (struct rq_hang_info *)src; + + detail_convert_to_json(dest, rq_hi); + sprintf(dest + strlen(dest), "%s", "\n"); + summary_convert_to_json(dest + strlen(dest), rq_hi); + sprintf(dest + strlen(dest), "%s", "\n"); +} + diff --git a/source/tools/detect/iosdiag/hangdetect/json_format.h b/source/tools/detect/iosdiag/hangdetect/json_format.h new file mode 100644 index 00000000..30ac3da2 --- /dev/null +++ b/source/tools/detect/iosdiag/hangdetect/json_format.h @@ -0,0 +1,9 @@ +#ifndef __JSON_FORMAT_H +#define __JSON_FORMAT_H + +#define JSON_BUFFER_SIZE (512*1024) + +void convert_to_json(char *dest, void *src); + +#endif + diff --git a/source/tools/detect/iosdiag/hangdetect/main.c b/source/tools/detect/iosdiag/hangdetect/main.c new file mode 100644 index 00000000..ae009e96 --- /dev/null +++ b/source/tools/detect/iosdiag/hangdetect/main.c @@ -0,0 +1,339 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "iosdiag.h" +#include "base_info.h" +#include "json_format.h" + +#define JSON_BUFFER_SIZE (512*1024) + +struct repeat_io { + unsigned long io_start; + unsigned long sector; +}; + +static struct repeat_io g_repeat_io[MAX_STORE_RQ_CNT]; +static int g_stop; +static int g_threshold = 5000; //5000ms +static int g_timeout = 10; //10s +static struct rq_hang_info *g_rq_hi; +static char *g_json_buf; +static int g_fd_res = -1; +static int g_fd_rq_hang_detect = -1; +static int g_bio_file_info; +static int g_once_capture_exit; +static char *g_result_dir = "/var/log/sysak/iosdiag/hangdetect"; +static char g_check_time_date[24]; + +static void detect_stop(int signo) +{ + printf("hang detect stop!\n"); + g_stop = 1; +} + +static void set_timeout_exit(void) +{ + signal(SIGINT, detect_stop); + signal(SIGALRM, detect_stop); + if (g_timeout) + alarm(g_timeout); +} + +static int exec_shell_cmd(char *cmd) +{ + char buf[128]; + FILE *fp; + + if (!cmd) + return -1; + + if ((fp = popen(cmd, "r")) == NULL) { + fprintf(stderr, "exec \'%s\' fail\n", cmd); + return -1; + } + + buf[sizeof(buf) - 1] = '\n'; + while (fgets(buf, sizeof(buf) - 1, fp)) + fprintf(stdout, "%s", buf); + pclose(fp); + return 0; +} + +static void set_check_time_date(void) +{ + struct timeval tv; + struct tm *p; + + gettimeofday(&tv, NULL); + p = localtime(&tv.tv_sec); + sprintf(g_check_time_date, "%d-%d-%d %d:%d:%d.%ld", + 1900+p->tm_year, + 1+p->tm_mon, + p->tm_mday, + p->tm_hour, + p->tm_min, + p->tm_sec, + tv.tv_usec / 1000); + set_base_info_check_time_date(g_check_time_date); +} + +static int trigger_rq_hang_collect(int fd, char *devname, int major, int minor, + int threshold) +{ + char devinfo[32] = {0}; + + if (major < 1 || minor < 0) { + fprintf(stderr, "invalid devnum(%d, %d)\n", major, minor); + return -1; + } + + sprintf(devinfo, "%s:%d:%d %d %d", devname, major, minor, threshold, g_bio_file_info); + set_check_time_date(); + if (write(fd, devinfo, strlen(devinfo) + 1) != (strlen(devinfo) + 1)) { + fprintf(stderr, "write devinfo \'%s\' fail(%s)\n", devinfo, strerror(errno)); + return -1; + } + return 0; +} + +static struct rq_hang_info *get_rq_hang_info_buffer(void) +{ + int fd; + void *p; + char *path = "/proc/disk_hang/rq_hang_detect"; + + fd = open(path, O_RDWR); + if (fd < 0) { + fprintf(stderr, "open \'%s\' fail(%s)\n" + "Please confirm the driver module is loaded\n", + path, strerror(errno)); + return NULL; + } + + p = mmap(NULL, (sizeof(struct rq_hang_info) * MAX_STORE_RQ_CNT), + PROT_READ, MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + fprintf(stderr, "mmap \'%s\' fail(%s)\n", path, strerror(errno)); + close(fd); + return NULL; + } + g_fd_rq_hang_detect = fd; + + return (struct rq_hang_info *)(p); +} + +static void put_rq_hang_info_buffer(struct rq_hang_info *buf) +{ + void *p; + + if (!buf) + return; + p = (void *)buf; + munmap(p, sizeof(struct rq_hang_info) * MAX_STORE_RQ_CNT); + if (g_fd_rq_hang_detect != -1) + close(g_fd_rq_hang_detect); +} + +static int open_result_file(char *filename) +{ + int fd; + char buf[256]; + char cmd[272]; + + if (strlen(filename) > (sizeof(buf) - 1)) { + printf("error: output file name(%s) too large(max %d bytes)\n", + filename, sizeof(buf)); + return -1; + } + strcpy(buf, filename); + sprintf(cmd, "mkdir %s -p", dirname(buf)); + exec_shell_cmd(cmd); + fd = open(filename, O_RDWR | O_CREAT, 0755); + if (fd < 0) { + printf("error: create output file \"%s\" fail\n", filename); + return -1; + } + return fd; +} + +static void write_result_file(int fd, char *buf, unsigned int len) +{ + write(fd, buf, len); +} + +static void close_result_file(int fd) +{ + if (fd > 0) + close(fd); +} + +static void usage(void) +{ + fprintf(stdout, + "\nUsage: \n" + "hangdetect [OPTION] disk_devname Detect IO hang in specified disk\n" + "hangdetect -t ms disk_devname Set IO hang threshold(default 5000ms)\n" + "hangdetect -T sec disk_devname How long to detect IO hang(default always)\n" + "hangdetect -f log disk_devname Specify the output file log\n" + "hangdetect -o disk_devname Auto exit once capture some ios\n" + "\ne.g.\n" + "hangdetect vda Detect IO hang in disk \"vda\"\n" + "hangdetect -t 1000 vda Set IO hang threshold 1000ms and detect IO hang in disk \"vda\"\n" + "hangdetect -t 1000 -o vda Detect IO hang delay for 1000ms in disk \"vda\" and auto exit once capture some ios\n" + "hangdetect -T 10 vda Detect IO hang in disk \"vda\" 10 secs\n"); + exit(-1); +} + +static int rq_existed_result_file(struct rq_hang_info *rq_hi, int idx) +{ + int existed = 0; + static unsigned int major = 0; + static unsigned int minor = 0; + + if (g_repeat_io[idx].io_start == rq_hi->io_start_ns && + g_repeat_io[idx].sector == rq_hi->sector && + major == get_bdi_major() && + minor == get_bdi_minor()) + existed = 1; + + g_repeat_io[idx].io_start = rq_hi->io_start_ns; + g_repeat_io[idx].sector = rq_hi->sector; + major = get_bdi_major(); + minor = get_bdi_minor(); + return existed; +} + +static void paser_arg(int argc, char *argv[]) +{ + int ch; + + if (argc < 2) + usage(); + + while ((ch = getopt(argc, argv, "T:t:f:heo")) != -1) { + switch (ch) { + case 'T': + g_timeout = (unsigned int)strtoul(optarg, NULL, 0); + break; + case 't': + g_threshold = (int)strtoul(optarg, NULL, 0); + break; + case 'f': + g_result_dir = optarg; + break; + case 'e': + g_bio_file_info = 1; + break; + case 'o': + g_once_capture_exit = 1; + break; + case 'h': + default: + usage(); + } + } +} + +static int run_collect(void) +{ + struct rq_hang_info *rq_hi = g_rq_hi; + char *json_buf = g_json_buf; + int fd_res = g_fd_res; + int current_bdi_idx = 0, muti_disk_collet = 0, capture_io = 0; + int i; + + if (get_current_bdi_idx() < 0) { + //check muti disk + set_current_bdi_idx(current_bdi_idx); + muti_disk_collet = 1; + } + + do { + if (!trigger_rq_hang_collect(g_fd_rq_hang_detect, + get_bdi_diskname(), + get_bdi_major(), + get_bdi_minor(), + g_threshold)) { + for (i = 0; i < MAX_STORE_RQ_CNT; i++) { + if (!rq_hi[i].req_addr) + continue; + if (rq_existed_result_file(&rq_hi[i], i)) + continue; + set_base_info_file(NULL); + convert_to_json(json_buf, &rq_hi[i]); + write_result_file(fd_res, json_buf, strlen(json_buf)); + capture_io = 1; + //printf(json_buf); + } + } else { + fprintf(stderr, "trigger \'%s\' rq_hang collect fail\n", + get_bdi_diskname()); + return -1; + } + current_bdi_idx++; + } while (muti_disk_collet && !set_current_bdi_idx(current_bdi_idx)); + + if (capture_io && g_once_capture_exit) + return 1; + + if (muti_disk_collet) + set_current_bdi_idx(-1); + + return 0; +} + +int main(int argc, char *argv[]) +{ + int ret = 0; + char resultfile_path[256]; + + paser_arg(argc, argv); + g_rq_hi = get_rq_hang_info_buffer(); + if (!g_rq_hi) { + fprintf(stderr, "get rq hang buffer fail\n"); + usage(); + } + + g_json_buf = malloc(JSON_BUFFER_SIZE); + if (!g_json_buf) { + fprintf(stderr, "malloc json_buf fail\n"); + ret = -1; + goto put_rq_buf; + } + memset(g_json_buf, 0x0, JSON_BUFFER_SIZE); + + sprintf(resultfile_path, "%s/result.log.seq", g_result_dir); + g_fd_res = open_result_file(resultfile_path); + if (g_fd_res < 0) { + fprintf(stderr, "create result file fail\n"); + ret = -1; + goto free_json_buf; + } + if (!base_info_init(argv[argc - 1])) { + set_timeout_exit(); + while (!g_stop) { + if (run_collect()) + break; + usleep(g_threshold / 2 * 1000); + } + } + base_info_exit(); +close_file: + close_result_file(g_fd_res); +free_json_buf: + free(g_json_buf); +put_rq_buf: + put_rq_hang_info_buffer(g_rq_hi); + return ret; +} + diff --git a/source/tools/detect/iosdiag/latency/main.c b/source/tools/detect/iosdiag/latency/main.c index 9ea70d9f..dd032d84 100644 --- a/source/tools/detect/iosdiag/latency/main.c +++ b/source/tools/detect/iosdiag/latency/main.c @@ -40,8 +40,9 @@ int main(int argc, char *argv[]) { int ch; int timeout_s = 10, threshold_ms = 1000; - char *result_file = "/var/log/sysak/iosdiag/latency/result.log"; + char *result_dir = "/var/log/sysak/iosdiag/latency"; char *devname; + char resultfile_path[256]; while ((ch = getopt(argc, argv, "T:t:f:h")) != -1) { switch (ch) { @@ -52,7 +53,7 @@ int main(int argc, char *argv[]) threshold_ms = (int)strtoul(optarg, NULL, 0); break; case 'f': - result_file = optarg; + result_dir = optarg; break; case 'h': default: @@ -65,7 +66,8 @@ int main(int argc, char *argv[]) fprintf(stderr, "iosdiag_init fail\n"); return -1; } - iosdiag_run(timeout_s, result_file); + sprintf(resultfile_path, "%s/result.log.seq", result_dir); + iosdiag_run(timeout_s, resultfile_path); iosdiag_exit(devname); return 0; } -- Gitee From 314788e22aa9a2e8475468fec489598d68907e76 Mon Sep 17 00:00:00 2001 From: "guangshui.li" Date: Mon, 28 Mar 2022 16:25:52 +0800 Subject: [PATCH 2/2] iosdiag: Fix the problem that 'target_devt' is not 0 when no disk is specified Signed-off-by: guangshui.li --- source/tools/detect/iosdiag/latency/collect.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/tools/detect/iosdiag/latency/collect.c b/source/tools/detect/iosdiag/latency/collect.c index 24b355ce..890c5fa1 100644 --- a/source/tools/detect/iosdiag/latency/collect.c +++ b/source/tools/detect/iosdiag/latency/collect.c @@ -153,7 +153,7 @@ static unsigned int get_devt_by_devname(char *devname) char cmd[128]; char dev[16]; FILE *fp; - int major, minor; + int major = 0, minor = 0; sprintf(sys_file, "/sys/block/%s/dev", devname); if (access(sys_file, F_OK)) -- Gitee