outline

fio is a very flexible io testing tool. It can simulate various io operations through multiple threads or processes

With the development of block devices, especially the emergence of SSD disks, the parallelism of devices is becoming higher and higher. To make good use of these devices, a trick is to improve the iodepth of the device, feed more IO requests to the device at one time, and give the elevator algorithm and device the opportunity to arrange merging and internal parallel processing, so as to improve the overall efficiency.

Applications usually use IO in two ways: synchronous and asynchronous. Synchronous IO can only send one IO request at a time and wait for the kernel to complete before returning. In this way, the iodepth for a single thread is always less than 1, but it can be solved by concurrent execution of multiple threads. Usually, we use 16-32 threads to work at the same time to fill the iodepth. In asynchronous mode, linux native aio such as libaio is used to submit a batch at a time, and then wait for the batch to complete. It will be more efficient to reduce the number of interactions.

Debugging environment

• Verification system
  loongnix1.0 version

• Verification tool version
  fio-3.12
  libaio-0.3.112

Problem description

On Godson platform, there is no problem using fio synchronous test, but asynchronous can not run, and the error is to call io_ The parameters passed by the submit interface are incorrect. It should be noted that fio asynchronously uses the libaio library, but actually uses the interface in the kernel.

Commissioning steps

Positioning from fio source code

TD was found by reporting the wrong location_ verror(td, -ret, "io commit");, Description TD - > IO_ ops->commit(td); Something's wrong
fio/ioengines.c

void td_io_commit(struct thread_data *td)
{
	int ret;

	dprint(FD_IO, "calling ->commit(), depth %d\n", td->cur_depth);

	if (!td->cur_depth || !td->io_u_queued)
		return;

	io_u_mark_depth(td, td->io_u_queued);

	if (td->io_ops->commit) {
		ret = td->io_ops->commit(td);
		if (ret)
			td_verror(td, -ret, "io commit");
	}

	/*
	 * Reflect that events were submitted as async IO requests.
	 */
	td->io_u_in_flight += td->io_u_queued;
	td->io_u_queued = 0;
}

Find function pointer TD - > IO_ ops->commit(td); The initialization place is determined to be static int FIO by printing_ libaio_ commit(struct thread_data *td)

fio/engines/libaio.c

static int fio_libaio_commit(struct thread_data *td)
{
	struct libaio_data *ld = td->io_ops_data;
	struct iocb **iocbs;
	struct io_u **io_us;
	struct timespec ts;
	int ret, wait_start = 0;

	if (!ld->queued)
		return 0;

	do {
		long nr = ld->queued;

		nr = min((unsigned int) nr, ld->entries - ld->tail);
		io_us = ld->io_us + ld->tail;
		iocbs = ld->iocbs + ld->tail;

		ret = io_submit(ld->aio_ctx, nr, iocbs);
		if (ret > 0) {
			fio_libaio_queued(td, io_us, ret);
			io_u_mark_submit(td, ret);

			ld->queued -= ret;
			ring_inc(ld, &ld->tail, ret);
			ret = 0;
			wait_start = 0;
		} else if (ret == -EINTR || !ret) {
			if (!ret)
				io_u_mark_submit(td, ret);
			wait_start = 0;
			continue;
		} else if (ret == -EAGAIN) {
			/*
			 * If we get EAGAIN, we should break out without
			 * error and let the upper layer reap some
			 * events for us. If we have no queued IO, we
			 * must loop here. If we loop for more than 30s,
			 * just error out, something must be buggy in the
			 * IO path.
			 */
			if (ld->queued) {
				ret = 0;
				break;
			}
			if (!wait_start) {
				fio_gettime(&ts, NULL);
				wait_start = 1;
			} else if (mtime_since_now(&ts) > 30000) {
				log_err("fio: aio appears to be stalled, giving up\n");
				break;
			}
			usleep(1);
			continue;
		} else if (ret == -ENOMEM) {
			/*
			 * If we get -ENOMEM, reap events if we can. If
			 * we cannot, treat it as a fatal event since there's
			 * nothing we can do about it.
			 */
			if (ld->queued)
				ret = 0;
			break;
		} else
			break;
	} while (ld->queued);

	return ret;
}

It is determined by printing_ There is something wrong with submit. Here is Io_ The prototype of submit is the macro definition interface. Check the interface in the libaio source code

#define io_syscall3(type,fname,sname,type1,arg1,type2,arg2,type3,arg3) \
type fname(type1 arg1,type2 arg2,type3 arg3) \
_body_io_syscall(sname, (long)arg1, (long)arg2, (long)arg3)

io_syscall3(int, io_submit, io_submit, io_context_t, ctx, long, nr, struct iocb **, iocbs)

Looking at the libaio interface, it is found that libaio implements the data structure, and the time interface calls the kernel
libaio-0.3.112/src/libaio.h

extern int io_setup(int maxevents, io_context_t *ctxp);
extern int io_destroy(io_context_t ctx);
extern int io_submit(io_context_t ctx, long nr, struct iocb *ios[]);
extern int io_cancel(io_context_t ctx, struct iocb *iocb, struct io_event *evt);
extern int io_getevents(io_context_t ctx_id, long min_nr, long nr, struct io_event *events, struct timespec *timeout);
extern int io_pgetevents(io_context_t ctx_id, long min_nr, long nr,

View AIO. Net in the kernel C as follows, it is found that the kernel data is incorrect

long do_io_submit(aio_context_t ctx_id, long nr,
		  struct iocb __user *__user *iocbpp, bool compat)
{
	struct kioctx *ctx;
	long ret = 0;
	int i = 0;
	struct blk_plug plug;
	printk("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!%d\n",__LINE__);
	printk(KERN_DEBUG "line%d,addr:%p,%p\n",__LINE__,*iocbpp,iocbpp);
	printk(KERN_DEBUG "line:%d,func:%s\n",__LINE__,__func__);
	if (unlikely(nr < 0))
		return -EINVAL;
	printk(KERN_DEBUG "line:%d,func:%s\n",__LINE__,__func__);
	if (unlikely(nr > LONG_MAX/sizeof(*iocbpp)))
		nr = LONG_MAX/sizeof(*iocbpp);

	printk(KERN_DEBUG "line:%d,func:%s\n",__LINE__,__func__);
	if (unlikely(!access_ok(VERIFY_READ, iocbpp, (nr*sizeof(*iocbpp)))))
		return -EFAULT;

	printk(KERN_DEBUG "line:%d,func:%s\n",__LINE__,__func__);
	ctx = lookup_ioctx(ctx_id);
	if (unlikely(!ctx)) {
		pr_debug("EINVAL: invalid context id\n");
		return -EINVAL;
	}

	printk(KERN_DEBUG "line:%d,func:%s\n",__LINE__,__func__);
	blk_start_plug(&plug);

	printk(KERN_DEBUG "line:%d,func:%s\n",__LINE__,__func__);
	/*
	 * AKPM: should this return a partial result if some of the IOs were
	 * successfully submitted?
	 */
	for (i=0; i<nr; i++) {
		struct iocb __user *user_iocb;
		struct iocb tmp;

		printk(KERN_DEBUG "line%d,addr:%p\n",__LINE__,user_iocb);
		printk(KERN_DEBUG "line%d,addr:%p\n",__LINE__,&tmp);
		if (unlikely(__get_user(user_iocb, iocbpp + i))) {
			ret = -EFAULT;
			break;
		}

		printk(KERN_DEBUG "line%d,addr:%p\n",__LINE__,user_iocb);
		if (unlikely(copy_from_user(&tmp, user_iocb, sizeof(tmp)))) {
			ret = -EFAULT;
			break;
		}
		printk(KERN_DEBUG "line%d,addr:%p-%d\n",__LINE__,&tmp,sizeof(tmp));

	printk(KERN_DEBUG "line:%d,func:%s\n",__LINE__,__func__);
		ret = io_submit_one(ctx, user_iocb, &tmp, compat);
		if (ret)
			break;
	}
	blk_finish_plug(&plug);

	put_ioctx(ctx);
	return i ? i : ret;
}

At the same time, print out the data in the kernel and the application for comparison, and find the problem. The data length in the kernel is 64 bytes, while the data length in the application is 83 bytes, so the data must not be right. Therefore, check the reason and find that it is a problem in libaio.

Problem solving

The libaio source code itself supports mips64el. In order to modify the support, you need to modify syscall in two places H and libaio h，

syscall. Add in H__ mips64 and syscall-mips64el h
__ mips64 is a macro defined in gcc

#if defined(__i386__)
#include "syscall-i386.h"
#elif defined(__x86_64__)
#include "syscall-x86_64.h"
#elif defined(__mips64)
#include "syscall-mips64el.h"
#elif defined(__ia64__)
#include "syscall-ia64.h"
#elif defined(__PPC__)
#include "syscall-ppc.h"
#elif defined(__s390__)
#include "syscall-s390.h"
#elif defined(__alpha__)
#include "syscall-alpha.h"
#elif defined(__arm__)
#include "syscall-arm.h"
#elif defined(__sparc__)
#include "syscall-sparc.h"
#elif defined(__aarch64__) || defined(__riscv)
#include "syscall-generic.h"
#else
#warning "using system call numbers from sys/syscall.h"
#endif

syscall-mips64el.h contents are as follows, from ~ ~ Linux / include / ASM arm / unistd h~~ /usr/include/asm/unistd. H

#define __NR_Linux                      5000
#define __NR_io_setup                   (__NR_Linux + 200)
#define __NR_io_destroy                 (__NR_Linux + 201)
#define __NR_io_getevents               (__NR_Linux + 202)
#define __NR_io_submit                  (__NR_Linux + 203)
#define __NR_io_cancel                  (__NR_Linux + 204)

libaio.h add the following at 64 bits__ mips64

/* little endian, 32 bits */
#if defined(__i386__) || (defined(__arm__) && !defined(__ARMEB__)) || \
    defined(__sh__) || defined(__bfin__) || (defined(__MIPSEL__) && defined(__mips32)) || \
    defined(__cris__) || (defined(__riscv) && __riscv_xlen == 32) || \
    (defined(__GNUC__) && defined(__BYTE_ORDER__) && \
         __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ && __SIZEOF_LONG__ == 4)
#define PADDED(x, y)    x; unsigned y
#define PADDEDptr(x, y) x; unsigned y
#define PADDEDul(x, y)  unsigned long x; unsigned y

/* little endian, 64 bits */
#elif defined(__ia64__) || defined(__x86_64__) || defined(__alpha__) ||         \
        (defined(__MIPSEL__) && defined(__mips64)) ||   \
      (defined(__aarch64__) && defined(__AARCH64EL__)) || \
      (defined(__riscv) && __riscv_xlen == 64) || \
      (defined(__GNUC__) && defined(__BYTE_ORDER__) && \
          __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ && __SIZEOF_LONG__ == 8)
#define PADDED(x, y)    x, y
#define PADDEDptr(x, y) x
#define PADDEDul(x, y)  unsigned long x

be careful:
In libaio The 32bit part of H is yes__ MIPSEL__ Judgment. We also have this macro in gcc, so we need to add other judgment to limit it, because both 32bit and 64bit have it__ MIPSEL__， Add__ mips32 macro is limited, and the mismatch between application and kernel data is the reason for this

Compile command

libaio

make clean;make ;make install

fio

make clean;make

Installation can be performed if required

make install

Verify problem command

fio -filename=/dev/sdb -direct=1 -iodepth=1 -thread -rw=write -ioengine=libaio -bs=4k -size=10G -numjobs=4 -runtime=120 -group_reporting -name=mytest