Well, never mind. I think I solved it by implementing doLayout in addition to runLayout. Here's the instance, if anyone is interested:

data EitherRatio p l a = EitherRatio (p a) (l a)
deriving (Show, Read)

instance (LayoutClass p a, LayoutClass l a) => LayoutClass (EitherRatio p l) a where
runLayout (W.Workspace i (EitherRatio p l) ms) r@(Rectangle rx ry rw rh)
  | rw < rh = do
      (lrs, newP) <- runLayout (W.Workspace i p ms) r
      return (lrs, fmap (`EitherRatio` l) newP)
  | rw >= rh = do
      (lrs, newL) <- runLayout (W.Workspace i l ms) r
      return (lrs, fmap (EitherRatio p) newL)

doLayout (EitherRatio p l) r@(Rectangle rx ry rw rh) s
  | rw < rh = do
      (lrs, newP) <- doLayout p r s
      return (lrs, fmap (`EitherRatio` l) newP)
  | rw >= rh = do
      (lrs, newL) <- doLayout l r s
      return (lrs, fmap (EitherRatio p) newL)

handleMessage (EitherRatio p l) msg = do
  mp <- handleMessage p msg
  ml <- handleMessage l msg
  return $ case (mp, ml) of
    (Just np, Just nl) -> Just $ EitherRatio np nl
    (Just np, Nothing) -> Just $ EitherRatio np l
    (Nothing, Just nl) -> Just $ EitherRatio p nl
    _                  -> Nothing

description (EitherRatio p l) =
  "[ " ++ description p ++ " | " ++ description l ++ " ]"