LibGL: Check for matrix stack over/underflow

We now correctly set the gloabl error if we detect that a matrix stack overflow will occur in `glPushMatrix` or `glPopMatrix`
mgaillard · May 8, 2021 · 55b3ecf · 55b3ecf
1 parent 1959efe
commit 55b3ecf
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 2 deletions.
diff --git a/Userland/Libraries/LibGL/GL/gl.h b/Userland/Libraries/LibGL/GL/gl.h
@@ -42,8 +42,10 @@ extern "C" {
 #define GL_INVALID_ENUM 0x500
 #define GL_INVALID_VALUE 0x501
 #define GL_INVALID_OPERATION 0x502
-#define GL_INVALID_FRAMEBUFFER_OPERATION 0x503
-#define GL_OUT_OF_MEMORY 0x504
+#define GL_STACK_OVERFLOW 0x0503
+#define GL_STACK_UNDERFLOW 0x0504
+#define GL_OUT_OF_MEMORY 0x505
+#define GL_INVALID_FRAMEBUFFER_OPERATION 0x506
 
 //
 // OpenGL typedefs

diff --git a/Userland/Libraries/LibGL/SoftwareGLContext.cpp b/Userland/Libraries/LibGL/SoftwareGLContext.cpp
@@ -48,6 +48,9 @@ enum ClippingPlane {
  FAR = 5
 };
 
+// FIXME: We should set this up when we create the context!
+static constexpr size_t MATRIX_STACK_LIMIT = 1024;
+
 // FIXME: Change this to accept a vertex!
 // Determines whether or not a vertex is inside the frustum for a given plane
 static bool vert_inside_plane(const FloatVector4& vec, ClippingPlane plane)
@@ -518,9 +521,17 @@ void SoftwareGLContext::gl_push_matrix()
 
  switch (m_current_matrix_mode) {
  case GL_PROJECTION:
+ if (m_projection_matrix_stack.size() >= MATRIX_STACK_LIMIT) {
+ m_error = GL_STACK_OVERFLOW;
+ return;
+ }
  m_projection_matrix_stack.append(m_projection_matrix);
  break;
  case GL_MODELVIEW:
+ if (m_model_view_matrix_stack.size() >= MATRIX_STACK_LIMIT) {
+ m_error = GL_STACK_OVERFLOW;
+ return;
+ }
  m_model_view_matrix_stack.append(m_model_view_matrix);
  break;
  default:
@@ -543,9 +554,17 @@ void SoftwareGLContext::gl_pop_matrix()
  // FIXME: Make sure stack::top() doesn't cause any nasty issues if it's empty (that could result in a lockup/hang)
  switch (m_current_matrix_mode) {
  case GL_PROJECTION:
+ if (m_projection_matrix_stack.size() == 0) {
+ m_error = GL_STACK_UNDERFLOW;
+ return;
+ }
  m_projection_matrix = m_projection_matrix_stack.take_last();
  break;
  case GL_MODELVIEW:
+ if (m_model_view_matrix_stack.size() == 0) {
+ m_error = GL_STACK_UNDERFLOW;
+ return;
+ }
  m_model_view_matrix = m_model_view_matrix_stack.take_last();
  break;
  default: