Side Quest: reading a model for hard edges

Situation:

  • heavy maya scene produces persistent artifacts not fixable by normal means

question for the void: does anyone know what causes this in autodesk #Maya?all i've done is combine two hipoly meshes into one, and the result has these degenerate rendering holessaving preserves the artifact; exporting works but bakes normal informationideally a fix in-situ would be nice but

[image or embed]

— STUAAAAAAAAAART (@stuaaaaaaaaaart.bsky.social) September 18, 2025 at 7:28 PM
  • exporting model removes artifacts but resets hard/soft-edge information
    • model is imported as all-hard-edges and all locked normals with all custom normal vectors (denoted in yellow instead of green in viewport display normal direction)
  • redoing hard/soft-edge information involves softening the model and visually going over model and selecting edge runs to harden them
    • extremely untenable for polygon-heavy or topologically very complex models

Script:

pretty much “get a model’s edges and check if the custom normals on each side are parallel or not. hard edges do not have parallel normals on either face sides

part 1: classes to condense all required openMaya operations into classes

import maya.cmds as mc
import maya.api.OpenMaya as om2

class edgeStu:
	def __init__(self, cDAG:om2.MDagPath, cID:int):
		self.DAG: om2.MDagPath = cDAG #om2, NOT string
		self.index = cID
		self.isSoft = False # models default to all-hard

		self.vtx  = [-1,-1]
		self.face = [-1,-1]
			# if more than 2: edge is non-manifold
			# this does not check if normal direction of one face is opposite of the other
				# no checks here, going off good faith polygon data for now
		# self.softNormalWeights = None # [0.5, 0.5] # do the theory next time, especially if weighted normal is right-angled to a soft edge corner
			# normalising function to weight soft normal direction later
		self.MFnEnumerate()

		pass
	def __str__(self):
		return f"{self.DAG.partialPathName()}.e[{self.index}]"
	def assertDAG(self):
		assert self.DAG.partialPathName() , f"edgeStu.DAG: DAG object not found or invalid"
		return self.DAG.partialPathName()
	def changeIndex(self, cID:int):
		numIndex = mc.polyEvaluate(self.DAG.partialPathName, e=True)  
		if (cID < numIndex) and (cID > -1):
			self.index = cID
			self.MFnEnumerate()
		else:
			raise IndexError(f"edgeStu.index: index out of range: range: 0-{numIndex-1}, got: {cID}")

	def MFnEnumerate(self):
		self.assertDAG()
		__MItEdge : om2.MItMeshEdge = om2.MItMeshEdge(self.DAG)
		__MItEdge.setIndex(self.index)
		__MItFace : om2.MItMeshPolygon = om2.MItMeshPolygon(self.DAG)
		
		self.vtx = [__MItEdge.vertexId(0), __MItEdge.vertexId(1)]
		self.face = list(__MItEdge.getConnectedFaces())
		# soft edge enumerator
		if len(self.face) < 2:
			self.isSoft = True
		elif len(self.face) == 2:
			__MItFV : om2.MItMeshFaceVertex = om2.MItMeshFaceVertex(self.DAG)

			self.isSoft = True # reset to assumed-soft for most DCCs
			vertList = [] # [[polyA vertlist], [polyB vertlist]]
			
			for face in self.face:
				__MItFace.setIndex(face)
				vertList.append(list(__MItFace.getVertices()))
			
			for vert in self.vtx: # range(len( a list that definitely has 2 elements ))
				__MItFV.setIndex(self.face[0], vertList[0].index(vert))
				faceVertNormA : om2.MVector = __MItFV.getNormal()
				__MItFV.setIndex(self.face[1], vertList[1].index(vert))
				faceVertNormB : om2.MVector = __MItFV.getNormal()
				if not faceVertNormA.isParallel(faceVertNormB): # if not parallel: hard edge
					self.isSoft = False
					break # doesn't matter if the second vert is parallel or not
			
			# cleanup, don't trust maya to collect itself
			del __MItFV
			del __MItEdge, __MItFace
		else:
			del __MItEdge, __MItFace
			raise ValueError(f"edgeStu.isSoft: nonManifold edge detected: edge borders not 1 or 2 faces")
		pass

	
class faceStu:
	def __init__(self, cDAG:om2.MDagPath, cID:int):
		self.DAG: om2.MDagPath = cDAG #om2, NOT string
		self.index = cID

		# below: if not 4: more is n-gon, 3 is triangle, 2 or less is degenerate
		self.vtx  = [-1,-1,-1,-1]
		self.edge = [-1,-1,-1,-1] # vertex 0-1, 1-2, 2-3, 3,0
		# self.uv   = [-1,-1,-1,-1]
		 	# NOTE: a model can have multiple UV sets
		# ensure all of them match according to vertex index
		self.MFnEnumerate()
		pass
	def __str__(self):
		return f"{self.DAG.partialPathName()}.f[{self.index}]"
	def assertDAG(self):
		assert self.DAG.partialPathName() , f"edgeStu.DAG: DAG object not found or invalid"
		return self.DAG.partialPathName()
	def changeIndex(self, cID:int):
		numIndex = mc.polyEvaluate(self.DAG.partialPathName, f=True)  
		if (cID < numIndex) and (cID > -1):
			self.index = cID
			self.MFnEnumerate()
		else:
			raise IndexError(f"faceStu.index: index out of range: range: 0-{numIndex-1}, got: {cID}")

	def MFnEnumerate(self):
		self.assertDAG()
		__MItFace : om2.MItMeshPolygon = om2.MItMeshPolygon(self.DAG)
		__MItFace.setIndex(self.index)
		
		self.vtx  = list(__MItFace.getVertices())
		self.edge = list(__MItFace.getEdges())
		
		# cleanup, don't trust maya to collect itself
		del __MItFace
		pass

part 2: actual script run:

# option 1: enter the name of the mesh
pMeshName = ""
pMeshMSL:om2.MSelectionList = om2.MSelectionList().add(pMeshName)

# option 2: select the mesh (selection sanity left as an exercise to the reader)
pMeshMSL:om2.MSelectionList = om2.MGlobal.getActiveSelectionList()

pMeshDAG:om2.MDagPath = pMeshMSL.getDagPath(0)

# ============= run this script
print("runtimee: this may take some time for very heavy models (think multiples of 10k)")
edgeList = [None] * mc.polyEvaluate(pMeshDAG.partialPathName(), e=True) 
faceList = [None] * mc.polyEvaluate(pMeshDAG.partialPathName(), f=True)

# load mesh
for faceIndex in range(len(faceList)):
	faceList[faceIndex] = faceStu(pMeshDAG, faceIndex)
	
	for edgeIndex in faceList[faceIndex].edge:
		# edgeStu.index ; skip if accessed previously
		if edgeList[edgeIndex] == None:
			edgeList[edgeIndex] = edgeStu(pMeshDAG, edgeIndex)

part 3: select hard edges

# select hard edges
edgeSoftMSL : om2.MSelectionList = om2.MSelectionList()

for edge in edgeList:
	if edge.isSoft:
		edgeSoftMSL.add(str(edge))

# mc.select(edgeSoftMSL.getSelectionStrings())
# fun note: the MSL will automatically concatenate selection strings in running order 

nAngleHard = 0
nAngleSoft = om2.MAngle(180, om2.MAngle.kDegrees).asUnits(om2.MAngle.uiUnit()) # the nuclear option, because angle attribute will depend on UI settings

# 0: hard, 180 or pi: soft
mc.UnlockNormals(pMeshDAG.partialPathName())
# imported model is default-hard
mc.polySoftEdge(edgeSoftMSL.getSelectionStrings(), a=nAngleSoft) # soften all soft edges

notes:

  • model’s normal selection/indexing is based on… face index and vertex index
  • script may not work well with coplanar sets of faces, as no bend is present for a hard edge to produce a non-parallel pair of normals

future additions

  • handling non-maya implementations of half-edge soft normals (where a “soft normal” is bent/biased toward a face instead of an average of either side of a set of faces)
    • requires understanding on topology intent of half-edge soft normal definitions (especially corner cases at 3-pole or 5-pole vertices), and how to numerically set custom normals