Platform: Quanzhi A133 Android Q

Start process

Data flow throughout the whole article

sensor0 ==> mipi0 ==> csi0 ==> isp0 ==> scale0(vipp0) ==> vinc0(dma0) ==> video0
sensor1 ==> mipi1 ==> csi1 ==> isp0 ==> scale1(vipp1) ==> vinc1(dma1) ==> video1

sensor, mipi, csi, isp, scale and vinc are all v4l2_subddev, corresponding to the / dev / v4l2 subdevx node,

Video is video_device, corresponding to the / dev/videoX node.

Quanzhi A133 command view

ceres-b3:/sys/devices/platform/soc@2900000/2000800.vind # cat vi
	*****************************************************
	VIN hardware feature list:
	mcsi 2, ncsi 1, parser 2, isp 1, vipp 4, dma 4
	CSI_VERSION: CSI300_100, ISP_VERSION: ISP522_100
	CSI_TOP: 336000000, CSI_ISP: 300000000
	*****************************************************
	vi0:
	ov13850_mipi => mipi0 => csi0 => isp0 => vipp0
	input => hoff: 0, voff: 0, w: 2112, h: 1568, fmt: BGGR10
	output => width: 1920, height: 1080, fmt: NV21
	interface: MIPI, isp_mode: NORMAL, hflip: 0, vflip: 0
	prs_in => x: 2112, y: 1568, hb: 3952, hs: 2766
	buf => cnt: 3 size: 3133440 rest: 3, mode: software_update
	frame => cnt: 1046, lost_cnt: 0, error_cnt: 0
	internal => avg: 32(ms), max: 33(ms), min: 32(ms)
	*****************************************************

Find the entrance first

Source location: longan/kernel/linux-4.9/drivers/media/platform/sunxi-vin/vin.c

static int __init vin_init(void)
	ret = sunxi_csi_platform_register();	//The corresponding probe() initializes the structure corresponding to csi, which contains v4l2_subdev,pads,ctrl, etc
	ret = sunxi_isp_platform_register();	//The corresponding probe() initializes the structure corresponding to csi, which contains v4l2_subdev,pads,ctrl, etc
	ret = sunxi_mipi_platform_register();	//The corresponding probe() initializes the structure corresponding to csi, which contains v4l2_subdev,pads,ctrl, etc
	ret = sunxi_flash_platform_register();	//The corresponding probe() initializes the structure corresponding to csi, which contains v4l2_subdev,pads,ctrl, etc
	ret = sunxi_scaler_platform_register();	//The corresponding probe() initializes the structure corresponding to csi, which contains v4l2_subdev,pads,ctrl, etc
	ret = sunxi_vin_core_register_driver();	//The corresponding probe() initializes the structure corresponding to csi, which contains v4l2_subdev,pads,ctrl, etc
	ret = platform_driver_register(&vin_driver);//General management, corresponding probe() registers video, v4l2_subdev,media, etc

Isn't it sensor v4l2_dev？ Take gc030a as an example:

Source location: longan/kernel/linux-4.9/drivers/media/platform/sunxi-vin/modules/sensor/gc030a_mipi.c

First look at the structure representing the sensor:

struct sensor_info {
	struct v4l2_subdev sd;
	struct media_pad sensor_pads[SENSOR_PAD_NUM];
	struct v4l2_ctrl_handler handler;
	... ...
};

The structures corresponding to mipi, csi, isp, scale and vinc also contain these structures. Continue to look at sensor's probe():

static int sensor_probe(struct i2c_client *client,const struct i2c_device_id *id)
	struct sensor_info *info;
	info = kzalloc(sizeof(struct sensor_info), GFP_KERNEL);
	cci_dev_probe_helper(sd, client, &sensor_ops, &cci_drv);
		v4l2_i2c_subdev_init(sd, client, sensor_ops);		//Initialize v4l2_subdev, setting v4l2_subdev_ops
		snprintf(sd->name, sizeof(sd->name), "%s", cci_drv->name);	//Set the name of subdev, here is gc030a_mipi
		v4l2_set_subdev_hostdata(sd, cci_drv);						//Private data saved as client
		sd->internal_ops = &sensor_internal_ops;					//Setup v4l2_subdev_internal_ops return function
			//==============================================================
			static const struct v4l2_subdev_internal_ops sensor_internal_ops = {
				.registered = sensor_registered,					//The s of the sensor will be called back here_ Power, init function
			};
			//==============================================================
	sensor_init_controls(sd, &sensor_ctrl_ops);						//Initialize v4l2_ control of subdev
	
	/* Other initialization */

Are there many callback functions? mipi, csi, isp, scale and vinc are also similar, mainly in vin callback. The following vin probe().

Source location: longan/kernel/linux-4.9/drivers/media/platform/sunxi-vin/vin.c

static const struct of_device_id sunxi_vin_match[] = {
	{.compatible = "allwinner,sunxi-vin-media",},  //==>Corresponding vind
	{},
};

static int vin_probe(struct platform_device *pdev)
	parse_modules_from_device_tree(vind);	//Get the device tree information and the sensor auto scan list
		parse_sensor_list_info(sensors, sensors->sensor_pos)
			sprintf(sensor_list_cfg, "/vendor/etc/hawkview/sensor_list_cfg.ini");
			cfg_section_init(&section);
			ret = cfg_read_ini(sensor_list_cfg, &section);
			if (strcmp(pos, "rear") == 0 || strcmp(pos, "REAR") == 0)
				fetch_sensor_list(sl, "rear_camera_cfg", section);
			else
				fetch_sensor_list(sl, "front_camera_cfg", section);
	ret = v4l2_device_register(dev, &vind->v4l2_dev);  v4l2 In fact, the device registration does not register the device or device driver, but v4l2 Large structures are bundled with other equipment
	media_device_init(&vind->media_dev);		//Management for runtime data flow
	ret = media_device_register(&vind->media_dev);		//Register the character device and generate / dev/mediaX
	ret = vin_md_register_entities(vind, dev->of_node){
		__vin_register_module(vind, module, j)
			modules->sensor[i].sd = __vin_subdev_register(vind, inst, inst->cam_name,inst->cam_addr >> 1,modules->sensor[i].type,module->sensors.sensor_bus_sel);
				struct i2c_adapter *adapter = i2c_get_adapter(bus_sel);
				sd = v4l2_i2c_new_subdev(v4l2_dev, adapter, name, addr, NULL);
					v4l2_i2c_new_subdev_board(v4l2_dev, adapter, &info, probe_addrs);
						client = i2c_new_device(adapter, info);
						sd = i2c_get_clientdata(client);				//Find the subdev instance from the private data of the client
						v4l2_device_register_subdev(v4l2_dev, sd)		//Register sensor subdev
							err = sd->internal_ops->registered(sd);	
								//Callback to longan/kernel/linux-4.9/drivers/media/platform/sunxi-vin/vin-cci/cci_helper.c
								static int sensor_registered(struct v4l2_subdev *sd)
									v4l2_subdev_call(sd, core, s_power, PWR_ON);		//Callback sensor_power
									ret = v4l2_subdev_call(sd, core, init, 0);			//Callback sensor_init
							list_add_tail(&sd->list, &v4l2_dev->subdevs);		//Just join the linked list
						
			
			//Autofocus subdev registration
			modules->act[i].sd = __vin_subdev_register(vind, inst, inst->act_name,inst->act_addr >> 1,modules->act[i].type,module->sensors.act_bus_sel);
		
		//Register subdev for flash
		ret = v4l2_device_register_subdev(&vind->v4l2_dev,module->modules.flash.sd);
		
		//Register vinc subdev and / dev/videoX
		for (i = 0; i < VIN_MAX_DEV; i++) {						//Here VIN_MAX_DEV is 4
			vind->vinc[i] = sunxi_vin_core_get_dev(i);			//From Vin_ core_ Get vinc from GBL array
			vin_md_register_core_entity(vind, vind->vinc[i]);	
				sd = &vinc->vid_cap.subdev;
				v4l2_set_subdev_hostdata(sd, (void *)&vin_pipe_ops);	//pipeline
				ret = v4l2_device_register_subdev(&vind->v4l2_dev, sd);
					if (sd->internal_ops && sd->internal_ops->registered)
						err = sd->internal_ops->registered(sd);		//Call back VIN of vinc here_ capture_ subdev_ Registered, register video_device
							vin_init_video(sd->v4l2_dev, &vinc->vid_cap)
								ret = video_register_device(&cap->vdev, VFL_TYPE_GRABBER, cap->vinc->id);	/// dev/videoX is generated
		}
		
		//Register CSI subdev
		ret = v4l2_device_register_subdev(&vind->v4l2_dev,vind->csi[i].sd);
		//Register mipi subdev
		ret = v4l2_device_register_subdev(&vind->v4l2_dev,vind->mipi[i].sd);
		//Register isp subdev
		ret = v4l2_device_register_subdev(&vind->v4l2_dev,vind->isp[i].sd);
		//Register scaler(vipp) subdev
		ret = v4l2_device_register_subdev(&vind->v4l2_dev,vind->scaler[i].sd);
	}
	
	//Last time I talked about v4l2_device_ register_ The subdev function simply adds subdev to v4l2_ In the device linked list, the real registration operation in batch will generate / dev / v4l2 subdevx
	ret = v4l2_device_register_subdev_nodes(&vind->v4l2_dev);

 }

entitys

entity in v4l2_subdev:

struct v4l2_subdev {
#if defined(CONFIG_MEDIA_CONTROLLER)
	struct media_entity entity;
#endif
    ... ...
}

entity in video_device:

struct video_device
{
#if defined(CONFIG_MEDIA_CONTROLLER)
	struct media_entity entity;
    ... ...
}

Thus, an entity represents a v4l2_subdev/video_device, a v4l2_subdev/video_device has only one entity.

pads

pads of sensor

Source code path: Drivers / media / platform / Sunxi VIN / modules / sensor / gc030a_ mipi.c

static int sensor_probe(struct i2c_client *client,const struct i2c_device_id *id)
└-> cci_dev_probe_helper(sd, client, &sensor_ops, &cci_drv);
	└-> ci_media_entity_init_helper(sd, cci_drv);
			switch (cci_drv->type) {
			case CCI_TYPE_SENSOR:		//sensor
				si->sensor_pads[SENSOR_PAD_SOURCE].flags = MEDIA_PAD_FL_SOURCE;		//Source, si is sensor_info
				sd->entity.function = MEDIA_ENT_F_CAM_SENSOR;
				return media_entity_pads_init(&sd->entity, SENSOR_PAD_NUM, si->sensor_pads);	//Here is SENSOR_PAD_NUM is 1 and there is only 1 pad, which is bound to entity
			case CCI_TYPE_ACT:			//focusing 
				sd->entity.function = MEDIA_ENT_F_LENS;
				return media_entity_pads_init(&sd->entity, 0, NULL);		//No pad
			case CCI_TYPE_FLASH:		//flash lamp
				sd->entity.function = MEDIA_ENT_F_FLASH;
				return media_entity_pads_init(&sd->entity, 0, NULL);		//No pad

• Line 6, sensor_pads are sensors_ An array size of info is SENSOR_PAD_NUM, SENSOR_PAD_NUM is 1;

• Line 8, media_entity_pads_init() binds the entity of pads and sensor s

Other Mipi, CSI, ISP and vinc are similar

Take isp as an example

longan/kernel/linux-4.9/drivers/media/platform/sunxi-vin/vin-isp/sunxi_isp.c

int __isp_init_subdev(struct isp_dev *isp)
	isp->isp_pads[ISP_PAD_SINK].flags = MEDIA_PAD_FL_SINK;			//objective
	isp->isp_pads[ISP_PAD_SOURCE_ST].flags = MEDIA_PAD_FL_SOURCE;	//source
	isp->isp_pads[ISP_PAD_SOURCE].flags = MEDIA_PAD_FL_SOURCE;		//source
	sd->entity.function = MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER;
	ret = media_entity_pads_init(&sd->entity, ISP_PAD_NUM, isp->isp_pads);	//ISP here_ pad_ Num is 3, which means there are 3 pads

links

vin_probe () --> vin_create_media_links

Source location: longan/kernel/linux-4.9/drivers/media/platform/sunxi-vin/vin.c

static int vin_probe(struct platform_device *pdev)
   media_device_init(&vind->media_dev);			//Initialize the linked lists of entities, pads, links, etc
   ret = media_device_register(&vind->media_dev);	//Register the character device and generate / dev/mediaX
   ret = vin_md_register_entities(vind, dev->of_node)
   	v4l2_device_register_subdev(... ...)
   		err = media_device_register_entity(v4l2_dev->mdev, entity);		//Entity, pad and link will be added to media_ Several linked lists of device
   
   ret = vin_create_media_links(vind);
   	for (i = 0; i < VIN_MAX_DEV; i++) {
   		vinc = vind->vinc[i];						//Find vinc
   		mipi = vind->mipi[vinc->mipi_sel].sd;		//Find mipi
   		csi = vind->csi[vinc->csi_sel].sd;			//Find csi
   		module = &vind->modules[vinc->rear_sensor];	//Find sensor
   			/*******************
   			 * The corresponding dts is
   			 *vinc0:vinc@0 {
   			 *	vinc0_csi_sel = <0>;
   			 *	vinc0_mipi_sel = <0>;
   			 *	vinc0_isp_sel = <0>;
   			 *	vinc0_isp_tx_ch = <0>;
   			 *	vinc0_tdm_rx_sel = <0xff>;
   			 *	vinc0_rear_sensor_sel = <0>;
   			 *	vinc0_front_sensor_sel = <1>;
   			 *	vinc0_sensor_list = <1>;
   			 *	status = "okay";
   			 *};
   			 ******************/
   		//Create link: sensor = = > Mipi
   		sensor_link_to_mipi_csi(module, mipi);
   			ret = media_create_pad_link(source, SENSOR_PAD_SOURCE, sink, 0, 0);
   		
   		//Create link: Mipi = = > CSI
   		source = &mipi->entity;
   		sink = &csi->entity;
   		ret = media_create_pad_link(source, MIPI_PAD_SOURCE,sink, CSI_PAD_SINK,MEDIA_LNK_FL_ENABLED);
   		
   		//Create link: scaler = = > vinc
   		source = &scaler->entity;
   		sink = &cap_sd->entity;
   		ret = media_create_pad_link(source, SCALER_PAD_SOURCE,sink, VIN_SD_PAD_SINK,MEDIA_LNK_FL_ENABLED);
   		
   		//Create link: vinc = = > Video
   		source = &cap_sd->entity;
   		sink = &vinc->vid_cap.vdev.entity;
   		ret = media_create_pad_link(source, VIN_SD_PAD_SOURCE,sink, 0, MEDIA_LNK_FL_ENABLED);	
   	}
   	
   	//Create link: CSI = = > ISP
   	for (i = 0; i < VIN_MAX_CSI; i++) {
   		csi = vind->csi[i].sd;
   		source = &csi->entity;
   		for (j = 0; j < VIN_MAX_ISP; j++) {		//Thus, each csi is connected to each isp
   			isp = vind->isp[j].sd;
   			sink = &isp->entity;
   			ret = media_create_pad_link(source, CSI_PAD_SOURCE,sink, ISP_PAD_SINK, 0);
   	}
   	
   	//Create link: ISP = = > scaler
   	for (i = 0; i < VIN_MAX_ISP; i++) {
   		isp = vind->isp[i].sd;
   		source = &isp->entity;
   		stat = vind->stat[i].sd;
   		sink = &stat->entity;
   		//There is one more creation link: ISP = = > stat
   		ret = media_create_pad_link(source, ISP_PAD_SOURCE_ST,sink, 0,MEDIA_LNK_FL_IMMUTABLE |MEDIA_LNK_FL_ENABLED);

   		for (j = 0; j < VIN_MAX_SCALER; j++) {
   			scaler = vind->scaler[j].sd;
   			sink = &scaler->entity;
   			ret = media_create_pad_link(source, ISP_PAD_SOURCE,sink, SCALER_PAD_SINK, 0);
   		}
   	}
   		
   		
   ret = vin_setup_default_links(vind);
   	for (i = 0; i < VIN_MAX_DEV; i++) {
           isp = vind->isp[vinc->isp_sel].sd;
           scaler = vind->scaler[vinc->vipp_sel].sd;
   		link = media_entity_find_link(&isp->entity.pads[ISP_PAD_SOURCE],&scaler->entity.pads[SCALER_PAD_SINK]);
   		ret = media_entity_setup_link(link, MEDIA_LNK_FL_ENABLED);
   		p = &vinc->vid_cap.pipe;
   		vin_md_prepare_pipeline(p, &vinc->vid_cap.vdev.entity);
   	}
   	
   ret = v4l2_device_register_subdev_nodes(&vind->v4l2_dev);
   	struct media_link *link;
   	link = media_create_intf_link(&sd->entity,&vdev->intf_devnode->intf,MEDIA_LNK_FL_ENABLED);

In Vin_ create_ media_ Calling media_ in links () create_ pad_ Link (), did you find that some of the last parameters are 0 and some are MEDIA_LNK_FL_ENABLED.

So Vin_ create_ media_ The map displayed after links () should look like this:

The gray arrow indicates that it has not been enable d

But, vin_setup_default_links() enables some ISPs = = > scale. Some changes have taken place in the links map, as follows:

Which isp to select is determined by the dts configuration of vincX. See the following vinc0:

vinc0:vinc@0 {
    vinc0_csi_sel = <0>;
    vinc0_mipi_sel = <0>;
    vinc0_isp_sel = <0>;		<====
    vinc0_isp_tx_ch = <0>;
    vinc0_tdm_rx_sel = <0xff>;
    vinc0_rear_sensor_sel = <0>;
    vinc0_front_sensor_sel = <1>;
    vinc0_sensor_list = <1>;
    status = "okay";
};

It can be seen from the above that the link of sensor = = > Mipi and CSI = = > ISP has not been enabled. Where is it enabled?

Source location: longan/kernel/linux-4.9/drivers/media/platform/sunxi-vin/vin-video/vin_video.c

static int vidioc_s_input(struct file *file, void *priv, unsigned int i)
	__vin_sensor_setup_link(vinc, module, valid_idx, 1)		//module is the sensor
		struct v4l2_subdev *sensor = module->modules.sensor[i].sd;
		subdev = vind->mipi[vinc->mipi_sel].sd;
		entity = &sensor->entity;
		list_for_each_entry(link, &entity->links, list)
		ret = media_entity_setup_link(link, MEDIA_LNK_FL_ENABLED);	//link enabling sensor = = > Mipi
		
	__csi_isp_setup_link(vinc, 1)
		csi = vind->csi[vinc->csi_sel].sd;
		isp = vind->isp[vinc->isp_sel].sd;
		link = media_entity_find_link(&csi->entity.pads[CSI_PAD_SOURCE],&isp->entity.pads[ISP_PAD_SINK]);
		ret = media_entity_setup_link(link, MEDIA_LNK_FL_ENABLED);

So the final links map looks like this:

ps: the green arrow indicates enabled, and the gray arrow indicates not enabled.

pipeline

pipeline initialization

pipeline is equivalent to the navigation line of a map. The starting point of navigation is sensorX and the end point is videoX

Let's think about it first. Where is the pipeline saved?

struct vin_core {
	... ...
	const struct vin_pipeline_ops *pipeline_ops;
	struct vin_vid_cap vid_cap;
};

struct vin_vid_cap {
    struct vin_pipeline pipe;
};

struct vin_pipeline {
	struct media_pipeline pipe;
	struct v4l2_subdev *sd[VIN_IND_MAX];
};

It can be seen that each vinc has a pipeline, Vin_ The pipeline stores the v4l2#u subdev that this "navigation line" passes through.

The calling process vin_probe() -- > vin_setup_default_links() -- > vin_md_prepare_pipeline() can be found above

vin_md_prepare_pipeline() is executed in a for loop

p = &vinc->vid_cap.pipe;
vin_md_prepare_pipeline(p, &vinc->vid_cap.vdev.entity);
	while (1) {
		struct media_pad *pad = NULL;

		/* Find remote source pad */
		for (i = 0; i < me->num_pads; i++) {
			struct media_pad *spad = &me->pads[i];
			if (!(spad->flags & MEDIA_PAD_FL_SINK))
				continue;
			pad = media_entity_remote_pad(spad);

		}
        if (pad == NULL)	//No pad found, exit
			break;
		sd = media_entity_to_v4l2_subdev(pad->entity);
		switch (sd->grp_id) {
		case VIN_GRP_ID_SENSOR:
			p->sd[VIN_IND_SENSOR] = sd;
			break;
		case VIN_GRP_ID_MIPI:
			p->sd[VIN_IND_MIPI] = sd;
			break;
		case VIN_GRP_ID_CSI:
			p->sd[VIN_IND_CSI] = sd;
			break;
		case VIN_GRP_ID_TDM_RX:
			p->sd[VIN_IND_TDM_RX] = sd;
			break;
		case VIN_GRP_ID_ISP:
			p->sd[VIN_IND_ISP] = sd;
			break;
		case VIN_GRP_ID_SCALER:
			p->sd[VIN_IND_SCALER] = sd;
			break;
		case VIN_GRP_ID_CAPTURE:
			p->sd[VIN_IND_CAPTURE] = sd;
			break;
		default:
			break;
		}
		me = &sd->entity;
	}

Line 11, look at the implementation of media_entity_remote_pad()

struct media_pad *media_entity_remote_pad(struct media_pad *pad)
{
	struct media_link *link;

	list_for_each_entry(link, &pad->entity->links, list) {
		if (!(link->flags & MEDIA_LNK_FL_ENABLED))
			continue;

		if (link->source == pad)
			return link->sink;

		if (link->sink == pad)
			return link->source;
	}

	return NULL;

}

In lines 5-14, only the link of MEDIA_LNK_FL_ENABLED will be returned.

It can be seen from the above that when the program runs here, some link s are not enable d. That is to say, the pipeline is not complete at this time. When will it be complete?

The pipeline of vinc0 should look like this:

p
|--sd[VIN_IND_SENSOR] 	= NULL
|--sd[VIN_IND_MIPI] 	= NULL
|--sd[VIN_IND_CSI] 		= NULL
|--sd[VIN_IND_ISP] 		= isp0
|--sd[VIN_IND_SCALER] 	= scale0
|--sd[VIN_IND_CAPTURE] 	= vinc0

Source location: longan/kernel/linux-4.9/drivers/media/platform/sunxi-vin/vin-video/vin_video.c

static int vidioc_s_input(struct file *file, void *priv, unsigned int i)
	__vin_sensor_setup_link(vinc, module, valid_idx, 1)		//module is the sensor
		struct v4l2_subdev *sensor = module->modules.sensor[i].sd;
		subdev = vind->mipi[vinc->mipi_sel].sd;
		entity = &sensor->entity;
		list_for_each_entry(link, &entity->links, list)
		ret = media_entity_setup_link(link, MEDIA_LNK_FL_ENABLED);	//link enabling sensor = = > Mipi
		
	__csi_isp_setup_link(vinc, 1)
		csi = vind->csi[vinc->csi_sel].sd;
		isp = vind->isp[vinc->isp_sel].sd;
		link = media_entity_find_link(&csi->entity.pads[CSI_PAD_SOURCE],&isp->entity.pads[ISP_PAD_SINK]);
		ret = media_entity_setup_link(link, MEDIA_LNK_FL_ENABLED);
	
	ret = vin_pipeline_call(vinc, open, &cap->pipe, &cap->vdev.entity, true);
		//The open callback _vin_pipeline_open
		static int __vin_pipeline_open(struct vin_pipeline *p,struct media_entity *me, bool prepare)
            vin_md_prepare_pipeline(p, me);		//Again
		

The pipeline of vinc0 should look like this:

p
|--sd[VIN_IND_SENSOR] 	= sensor0
|--sd[VIN_IND_MIPI] 	= mipi0
|--sd[VIN_IND_CSI] 		= csi0
|--sd[VIN_IND_ISP] 		= isp0
|--sd[VIN_IND_SCALER] 	= scale0
|--sd[VIN_IND_CAPTURE] 	= vinc0

Use of pipeline

After working hard to build a pipeline, it must be used.

It seems that this SDK does not use a function like media_entity_pipeline_start().

The use of pipeline is also simple and rough, such as:

v4l2_subdev_call(cap->pipe.sd[VIN_IND_ISP], core, init, 0);

Directly call the callback function of subdev.

When using vidic_s_input or vidic_streamon, you can clearly know which subdev to operate through the pipe array in vid_cap.

finish

The above is the author's learning notes. If you find something wrong, please don't hesitate to give advice. Thank you.