Experience of adapting vtk window to touch screen with Qt
The recent work arrangement is to adapt vtk's touch screen operation. After reading some source code, I share my current methods here.
This article is my own idea, only for reference, may be completely wrong.!!! If there are other elegant ways, please correct.
original text
Experience of adapting vtk window to touch screen with Qt
Thank you for sharing https://github.com/tsutenn/TsutennQVTKWidget , after understanding his idea, I rewritten it.
A simple Demo supporting multi finger touch operation vtp (single finger rotation, double finger zoom, three finger translation):
https://github.com/BeyondXinXin/study_vtk (the warehouse is very messy, so it's open source)
According to their own testing and integration process, they are roughly divided into five parts:
- qvtk receive touch event
- Interactive modification of custom Rep
- Interactive modification of vtkeabstractwidget and its subclasses
- "Multi touch" interactive support for pictures and models (single finger rotation, double finger zoom, three finger translation, etc.)
- Picture and model "button + single finger" interactive support (rotation, translation, zoom, window width, window level, transparency, reset, etc.)
qvtk touch support
In fact, the QVTKInteractorAdapter in the Vtk source code has added touch screen operation judgment, but the corresponding vtkCommand has not been added. MT (multi finger touch) operation is directly changed to:
Qt::TouchPointReleased ==> vtkCommand::LeftButtonReleaseEvent
Qt::TouchPointPressed ==> vtkCommand::LeftButtonPressEvent
Qt::TouchPointMoved ==> vtkCommand::MouseMoveEvent
Speechless, if the official directly increases the corresponding vtkCommand and camera operation, it can save a lot of things.
In order to adapt the touch operation, I think of the following methods:
Implement a TouchInteractor to replace the QVTKInteractorAdapter. Bind Qt events and TouchInteractor with Qt signal slot, and distribute them to other reps and camera operations with vtk observe after parsing.
In order to achieve asynchrony, TouchInteractor needs to maintain Qt event queue and distribute events from the queue in batches by using vtkwindow interactor's RepeatingTimer.
Steps:
- Turn on qvtkxxwidget's touch screen support (Qt::WA_AcceptTouchEvents, Qt::WA_WState_AcceptedTouchBeginEvent, Qt::WA_TouchPadAcceptSingleTouchEvents)
- The event filter that intercepts qvtkxxwidget is given to the custom Interactor to operate the camera and Rep
- All customized reps and built-in reps are rewritten: increase the picker range (fingers are not easy to select), and bind multi finger operations to the original keyboard, middle key, scroll wheel and other operations
Bind Qt events and TouchInteractor with Qt's signal slot
struct TsutennTask
{
TsutennTask(int t, QList<QPointF> l)
{
this->type_ = t;
this->points_ = l;
}
int type_ = -1;
QList<QPointF> points_;
};
class SlotHelper : public QObject
{
Q_OBJECT
private:
QQueue<TsutennTask *> tasks;
public:
SlotHelper(TouchRenderWidget *parent = Q_NULLPTR);
~SlotHelper() override;
bool HasTask();
TsutennTask *GetTask();
public slots:
void TouchBeginEventProcessor(QList<QPointF> points);
void TouchUpdateEventProcessor(QList<QPointF> points);
void ToucEndEventProcessor();
void MouseBeginEventProcessor(QPointF points);
void MouseEndEventProcessor(QPointF startpos, QPointF endpos);
void MouseDoubleClickEventProcessor(QPointF clickpos);
void MouseMoveEventProcessor(QPointF movepos);
};
SlotHelper::SlotHelper(TouchRenderWidget *parent)
{
void (TouchRenderWidget::*sgn1)(QList<QPointF>) = &TouchRenderWidget::SgnTouchBegin;
void (TouchRenderWidget::*sgn2)(QList<QPointF>) = &TouchRenderWidget::SgnTouchUpdata;
void (TouchRenderWidget::*sgn3)(void) = &TouchRenderWidget::SgnTouchEdn;
void (TouchRenderWidget::*sgn4)(QPointF) = &TouchRenderWidget::SgnMouseBegin;
void (TouchRenderWidget::*sgn5)(QPointF, QPointF) = &TouchRenderWidget::SgnMouseEnd;
void (TouchRenderWidget::*sgn6)(QPointF) = &TouchRenderWidget::SgnMouseDoubleClick;
void (TouchRenderWidget::*sgn7)(QPointF) = &TouchRenderWidget::SgnMouseMove;
void (SlotHelper::*slot1)(QList<QPointF>) = &SlotHelper::TouchBeginEventProcessor;
void (SlotHelper::*slot2)(QList<QPointF>) = &SlotHelper::TouchUpdateEventProcessor;
void (SlotHelper::*slot3)(void) = &SlotHelper::ToucEndEventProcessor;
void (SlotHelper::*slot4)(QPointF) = &SlotHelper::MouseBeginEventProcessor;
void (SlotHelper::*slot5)(QPointF, QPointF) = &SlotHelper::MouseEndEventProcessor;
void (SlotHelper::*slot6)(QPointF) = &SlotHelper::MouseDoubleClickEventProcessor;
void (SlotHelper::*slot7)(QPointF) = &SlotHelper::MouseMoveEventProcessor;
QObject::connect(parent, sgn1, this, slot1);
QObject::connect(parent, sgn2, this, slot2);
QObject::connect(parent, sgn3, this, slot3);
QObject::connect(parent, sgn4, this, slot4);
QObject::connect(parent, sgn5, this, slot5);
QObject::connect(parent, sgn6, this, slot6);
QObject::connect(parent, sgn7, this, slot7);
}
SlotHelper::~SlotHelper()
{
QObject::disconnect(this);
}
bool :SlotHelper::HasTask()
{
return !this->tasks.isEmpty();
}
TsutennTask *SlotHelper::GetTask()
{
return this->tasks.dequeue();
}
Redistributed TouchInteractor
class TouchRenderCallback : public vtkCommand
{
public:
enum State
{
kNone,
kRotate,
kDolly,
kPan,
KTransparency,
kWindowLevel,
};
public:
static TouchRenderCallback *New(TouchRenderWidget *parent = Q_NULLPTR);
vtkTypeMacro(TouchRenderCallback, vtkCommand);
void SetParent(RenderFrame *parent);
void Execute(vtkObject *caller, unsigned long vtkNotUsed(eventId), void *vtkNotUsed(callData)) override;
void SetState(const State &state);
vtkSmartPointer<vtkRenderer> render_;
vtkSmartPointer<vtkRenderWindow> ren_win_;
void SetVolume(const std::shared_ptr<VolumeRepresentation> &vol_repr);
private:
virtual void TouchBeginExecute(QList<QPointF>);
virtual void TouchUpdateExecute(QList<QPointF>);
virtual void TouchEndExecute(QList<QPointF>);
virtual void MouseBeginExecute(QList<QPointF>);
virtual void MouseEndExecute(QList<QPointF>);
virtual void MouseDoubleClickExecute(QList<QPointF>);
virtual void MouseMoveExecute(QList<QPointF>);
private:
void CamRotate(int delta[], const double &rotation_angle = 0.2);
void CamSecurityDolly(int delta[], double factor = 10);
void CamPan(const QPoint &pos1, const QPoint &pos2);
void ShiftWindow(int delta[]);
private:
std::shared_ptr<VolumeRepresentation> vol_repr_;
SlotHelper *callback_slot_;
RenderFrame *parent_;
State state_ { kNone };
QList<QPointF> last_mouse_;
QList<QPointF> last_touch_;
qreal zoom_para_;
};
void TouchRenderCallback::Execute(
vtkObject *vtkNotUsed(caller), unsigned long vtkNotUsed(eventId), void *vtkNotUsed(callData))
{
while (this->callback_slot_->HasTask()) {
TsutennTask *task = this->callback_slot_->GetTask();
switch (task->type_) {
case 0:
qDebug() << "Touch begin";
TouchBeginExecute(task->points_);
break;
case 1:
TouchUpdateExecute(task->points_);
break;
case 2:
qDebug() << "Touch End";
TouchEndExecute(task->points_);
break;
case 3:
qDebug() << "Mouse press";
MouseBeginExecute(task->points_);
break;
case 4:
qDebug() << "Mouse release";
MouseEndExecute(task->points_);
break;
case 5:
qDebug() << "Mouse double click";
MouseDoubleClickExecute(task->points_);
break;
case 6:
MouseMoveExecute(task->points_);
break;
}
}
ren_win_->Render();
}
Blocked qvtkxxwidget
bool TouchRenderWidget::eventFilter(QObject *object, QEvent *event)
{
if (!use_touch_callback_) {
if (event->type() == QEvent::Wheel) {
return true;
}
return RenderWidget::eventFilter(object, event);
}
if (event->type() == QEvent::TouchBegin) {
return TouchBeginEventProcess(event);
} else if (event->type() == QEvent::TouchUpdate) {
return TouchUpdateEventProcess(event);
} else if (event->type() == QEvent::TouchEnd) {
return TouchEndEventProcess(event);
} else if (event->type() == QEvent::MouseButtonPress) {
return MousePressEventProcess(event);
} else if (event->type() == QEvent::MouseButtonRelease) {
return MouseReleaseEventProcess(event);
} else if (event->type() == QEvent::MouseButtonDblClick) {
return MouseDoubleClickEventProcess(event);
} else if (event->type() == QEvent::MouseMove) {
return MouseMoveEventProcess(event);
} else if (event->type() == QEvent::Wheel) {
return true;
}
return RenderWidget::eventFilter(object, event);
}
bool TouchRenderWidget::TouchBeginEventProcess(QEvent *event)
{
QTouchEvent *touchEvent = static_cast<QTouchEvent *>(event);
touch_points_ = touchEvent->touchPoints();
QList<QPointF> points;
for (int i = 0; i < touch_points_.count(); i++) {
points.append(touch_points_.at(i).pos());
}
emit SgnTouchBegin(points);
return true;
}
bool TouchRenderWidget::TouchUpdateEventProcess(QEvent *event)
{
QTouchEvent *touchEvent = static_cast<QTouchEvent *>(event);
touch_points_ = touchEvent->touchPoints();
QList<QPointF> points;
for (int i = 0; i < touch_points_.count(); i++) {
points.append(touch_points_.at(i).pos());
}
emit SgnTouchUpdata(points);
return true;
}
bool TouchRenderWidget::TouchEndEventProcess(QEvent *)
{
touch_points_.clear();
SgnTouchEdn();
return true;
}
bool TouchRenderWidget::MousePressEventProcess(QEvent *event)
{
QMouseEvent *mouse_event = static_cast<QMouseEvent *>(event);
if (mouse_event->buttons().testFlag(Qt::MidButton)
|| mouse_event->buttons().testFlag(Qt::RightButton)) {
return true;
}
if (touch_points_.isEmpty()) {
QMouseEvent *mouse_event = static_cast<QMouseEvent *>(event);
start_pos_ = mouse_event->pos();
emit SgnMouseBegin(start_pos_);
process_mouse_event_ = true;
}
return true;
}
bool TouchRenderWidget::MouseReleaseEventProcess(QEvent *event)
{
if (process_mouse_event_) {
process_mouse_event_ = false;
QMouseEvent *mouse_event = static_cast<QMouseEvent *>(event);
end_pos_ = mouse_event->pos();
emit SgnMouseEnd(start_pos_, end_pos_);
}
return true;
}
bool rtx::TouchRenderWidget::MouseDoubleClickEventProcess(QEvent *event)
{
QMouseEvent *mouse_event = static_cast<QMouseEvent *>(event);
QPointF Pos = mouse_event->pos();
emit SgnMouseDoubleClick(Pos);
return true;
}
bool rtx::TouchRenderWidget::MouseMoveEventProcess(QEvent *event)
{
if (process_mouse_event_) {
QMouseEvent *mouse_event = static_cast<QMouseEvent *>(event);
QPointF pos = mouse_event->pos();
emit SgnMouseMove(pos);
}
return true;
}
Interactive modification of custom Rep
Interactive modification of vtkeabstractwidget and its subclasses
Whether it is a custom Rep or a vtkAbstractWidget, basically all vtk classes used need to be rewritten.
At present, only some changes have been made, and the road ahead is slow. for instance:
vtkPropPicker needs to gradually expand the selection range until the target Prop is found. The maintenance of vtkPropCollection should also be added.
class ForkePropPicker : public vtkPropPicker
{
public:
static ForkePropPicker *New();
vtkTypeMacro(ForkePropPicker, vtkAbstractPropPicker);
int Pick(double selectionX, double selectionY, double selectionZ,
vtkRenderer *renderer) override;
int PickProp(double selectionX, double selectionY, vtkRenderer *renderer,
vtkPropCollection *pickfrom);
};
vtkStandardNewMacro(ForkePropPicker);
int ForkePropPicker::Pick(double selectionX, double selectionY, double, vtkRenderer *renderer)
{
this->Initialize();
double offset = 5.0;
int number_enlarge = 50;
while (!this->Path && number_enlarge > 1) {
// Initialize picking process
offset += 1.0;
number_enlarge--;
this->Initialize();
this->Renderer = renderer;
this->SelectionPoint[0] = selectionX;
this->SelectionPoint[1] = selectionY;
this->SelectionPoint[2] = 0;
// If defined, the start pick method is called
this->InvokeEvent(vtkCommand::StartPickEvent, nullptr);
// Let the renderer perform a hardware pick
this->SetPath(renderer->PickPropFrom(selectionX - offset, selectionY - offset,
selectionX + offset, selectionY + offset, this->PickFromProps));
// If there is a pick, find the picked World x, y, z and call its pick method.
if (this->Path) {
this->WorldPointPicker->Pick(selectionX, selectionY, 0, renderer);
this->WorldPointPicker->GetPickPosition(this->PickPosition);
this->Path->GetLastNode()->GetViewProp()->Pick();
this->InvokeEvent(vtkCommand::PickEvent, nullptr);
}
this->InvokeEvent(vtkCommand::EndPickEvent, nullptr);
}
// Call Pick on the picked props and return 1 to indicate success
if (this->Path) {
return 1;
} else {
return 0;
}
}
int ForkePropPicker::PickProp(double selectionX, double selectionY, vtkRenderer *renderer, vtkPropCollection *pickfrom)
{
this->PickFromProps = pickfrom;
int ret = this->Pick(selectionX, selectionY, 0, renderer);
this->PickFromProps = nullptr;
return ret;
}
"Multi touch" interactive support for pictures and models (single finger rotation, double finger zoom, three finger translation, etc.)
This is to modify the camera operation. At present, the judgment method is relatively simple. Double finger up and down, left and right sliding and other operations can add judgment here
void TouchRenderCallback::TouchUpdateExecute(QList<QPointF> points)
{
if (kNone == state_) {
return;
}
if (points.size() == 1 || last_touch_.size() != points.size()) {
auto tmp = (points.at(0) - last_touch_.at(0)).toPoint();
int delta[2] = { tmp.x(), -tmp.y() };
if (kRotate == state_) {
CamRotate(delta);
} else if (kDolly == state_) {
CamSecurityDolly(delta);
} else if (kPan == state_) {
CamPan(last_touch_.at(0).toPoint(), points.at(0).toPoint());
} else if (KTransparency == state_) {
ShiftWindow(delta);
} else if (kWindowLevel == state_) {
int deltax_, deltay_;
deltax_ = delta[0];
deltay_ = delta[1];
parent_->SgnWindowLevelChange(deltax_, deltay_);
}
} else if (points.size() == 2) {
auto tmp = (points.at(0) - last_touch_.at(0)).toPoint();
int delta[2] = { tmp.x(), -tmp.y() };
CamSecurityDolly(delta);
} else if (points.size() == 3) {
CamPan(last_touch_.at(0).toPoint(), points.at(0).toPoint());
} else if (points.size() == 4) {
auto tmp = (points.at(0) - last_touch_.at(0)).toPoint();
int delta[2] = { tmp.x(), -tmp.y() };
ShiftWindow(delta);
}
last_touch_ = points;
}
About rewriting the camera operation, take a direct look at the implementation of vtk's own interaction, and change it accordingly. For example, zoom:
void TouchRenderCallback::CamSecurityDolly(int delta[], double factor)
{
int dy = delta[1];
double *center = this->render_->GetCenter();
double dyf = factor * dy / center[1];
dyf = pow(1.1, dyf);
vtkCamera *camera = this->render_->GetActiveCamera();
if (camera->GetParallelProjection()) {
auto par = camera->GetParallelScale();
if (par + 1e-6 < 10.0 && dyf + 1e-6 > 1.0) {
return;
} else if (par - 1e-6 > 200.010 && dyf + 1e-6 < 1.0) {
return;
}
camera->SetParallelScale(camera->GetParallelScale() / dyf);
} else {
auto dis = camera->GetDistance();
if (dis + 1e-6 < 100.0 && dyf + 1e-6 > 1.0) {
return;
} else if (dis - 1e-6 > 3000.010 && dyf + 1e-6 < 1.0) {
return;
}
camera->Zoom(dyf);
}
if (this->render_->GetLightFollowCamera()) {
this->render_->UpdateLightsGeometryToFollowCamera();
}
render_->Render();
}
Picture and model "button + single finger" interactive support (rotation, translation, zoom, window width, window level, transparency, reset, etc.)
Make several mutually exclusive buttons to realize different adjustments
void RenderFrame::AddTouchButton(FrameState frame_state)
{
auto FunGenBtn = [&](const QString &str, const QString &objstr, const State &state, const bool &checkable = false) {
QPushButton *btn = new QPushButton(widget_);
btn->setFixedSize(100, 100);
btn->setText(str);
btn->setObjectName(objstr);
btn->setProperty("CustomStyle", "4view");
btn->setProperty("attribute", state);
btn->setCheckable(checkable);
connect(btn, &QPushButton::clicked, this, [=]() {
BtnClicked(State(QObject::sender()->property("attribute").toInt()));
});
return btn;
};
if (frame_state == Frame_3d) {
btn_group_->addButton(FunGenBtn(u8"rotate", u8"route", I_Rotate, true), 0);
btn_group_->addButton(FunGenBtn(u8"zoom", u8"dolly", I_Dolly, true), 1);
btn_group_->addButton(FunGenBtn(u8"translation", u8"Pan", I_Pan, true), 2);
btn_group_->addButton(FunGenBtn(u8"transparent", u8"Transparency", I_Transparency, true), 3);
btn_group_->addButton(FunGenBtn(u8"Camera reset", u8"Transparency", I_CamReset), 4);
btn_group_->addButton(FunGenBtn(u8"Toggle color", u8"ChangeVrClor", W_ChangeVrClor), 5);
btn_group_->addButton(FunGenBtn(u8"Render Mode ", u8"ChangeVrModel", W_ChangeVrModel), 6);
btn_group_->addButton(FunGenBtn(u8"Transparent reset", u8"TranReset", W_TranReset), 7);
widget_->SetUsrTouchCallback(true);
} else if (frame_state == Frame_2d) {
btn_group_->addButton(FunGenBtn(u8"position", u8"Pos", I_Pos, true), 0);
btn_group_->addButton(FunGenBtn(u8"sequence", u8"Order", I_Order, true), 1);
btn_group_->addButton(FunGenBtn(u8"zoom", u8"dolly", I_Dolly, true), 2);
btn_group_->addButton(FunGenBtn(u8"translation", u8"Pan", I_Pan, true), 3);
btn_group_->addButton(FunGenBtn(u8"Window width level", u8"WindowLevel", I_WindowLevel, true), 4);
btn_group_->addButton(FunGenBtn(u8"Camera reset", u8"Transparency", I_CamReset), 5);
btn_group_->addButton(FunGenBtn(u8"Window width reset", u8"WLReset", W_WLReset), 6);
widget_->SetUsrTouchCallback(false);
}
btn_group_->button(0)->click();
}
void RenderFrame::BtnClicked(State state)
{
switch (state) {
case I_None: {
iren_refresh_->SetState(TouchRenderCallback::kNone);
widget_->SetUsrTouchCallback(true);
} break;
case I_Rotate: {
iren_refresh_->SetState(TouchRenderCallback::kRotate);
widget_->SetUsrTouchCallback(true);
} break;
case I_Dolly: {
iren_refresh_->SetState(TouchRenderCallback::kDolly);
widget_->SetUsrTouchCallback(true);
} break;
case I_Pan: {
iren_refresh_->SetState(TouchRenderCallback::kPan);
widget_->SetUsrTouchCallback(true);
} break;
case I_Transparency: {
iren_refresh_->SetState(TouchRenderCallback::KTransparency);
widget_->SetUsrTouchCallback(true);
} break;
case I_WindowLevel: {
iren_refresh_->SetState(TouchRenderCallback::kWindowLevel);
widget_->SetUsrTouchCallback(true);
} break;
case I_Pos: {
widget_->SetUsrTouchCallback(false);
emit SgnFrameStateChange(state);
} break;
case I_Order: {
widget_->SetUsrTouchCallback(false);
emit SgnFrameStateChange(state);
} break;
case I_CamReset:
case W_WLReset:
case W_TranReset:
case W_ChangeVrClor:
case W_ChangeVrModel: {
emit SgnFrameStateChange(state);
} break;
}
}
void Interactor3DAction::Move(InteractEvent *e, bool self)
{
interce_event_ = e;
int tem_delta[2] = { (e->Position() - e->LastPosition())[0], (e->Position() - e->LastPosition())[1] };
switch (this->state_) {
case kRotate:
CamRotate(tem_delta);
if (self) {
Invoke<Rotate>();
}
break;
case kSpin:
track_ball_->Spin(e->Position(), e->LastPosition());
if (self) {
Invoke<Spin>();
}
break;
case kDolly:
track_ball_->SecurityDolly(e->Position() - e->LastPosition(), motion_factor_);
if (self) {
Invoke<Dolly>();
}
break;
case KWheelForward:
track_ball_->SecurityDolly(pow(1.1, motion_factor_ * 0.2));
if (self) {
Invoke<WheelForward>();
}
break;
case KWheelBackward:
track_ball_->SecurityDolly(pow(1.1, -motion_factor_ * 0.2));
if (self) {
Invoke<WheelBackward>();
}
break;
case kPan:
track_ball_->Pan(e->Position(), e->LastPosition());
if (self) {
Invoke<Pan>();
}
break;
case kWindowLevel:
ShiftWindow(e->Renderer(), e->Position() - e->LastPosition(), motion_factor_);
if (self) {
Invoke<WindowLevel>();
}
break;
default:
return;
}
e->RequestRender();
}