This experimental patch limits the latency that can be introduced by 
increase_watermark().  It's helpful for watching TV via tvtime, but there is 
probably a better way of solving the problem.

--- src/modules/alsa/alsa-source.c.orig	2012-05-12 22:30:59.335942782 -0500
+++ src/modules/alsa/alsa-source.c	2012-05-18 23:43:49.140383432 -0500
@@ -281,6 +281,13 @@
 
     if (u->tsched_watermark < u->min_wakeup)
         u->tsched_watermark = u->min_wakeup;
+
+    /* Set the maximum watermark to 50 msec. */
+    size_t max_watermark = pa_usec_to_bytes(50*PA_USEC_PER_MSEC, &u->source->sample_spec);
+    if (u->tsched_watermark > max_watermark)
+    {
+        u->tsched_watermark = max_watermark;
+    }
 }
 
 static void increase_watermark(struct userdata *u) {
@@ -301,17 +308,7 @@
         return;
     }
 
-    /* Hmm, we cannot increase the watermark any further, hence let's raise the latency */
-    old_min_latency = u->source->thread_info.min_latency;
-    new_min_latency = PA_MIN(old_min_latency * 2, old_min_latency + TSCHED_WATERMARK_INC_STEP_USEC);
-    new_min_latency = PA_MIN(new_min_latency, u->source->thread_info.max_latency);
-
-    if (old_min_latency != new_min_latency) {
-        pa_log_info("Increasing minimal latency to %0.2f ms",
-                    (double) new_min_latency / PA_USEC_PER_MSEC);
-
-        pa_source_set_latency_range_within_thread(u->source, new_min_latency, u->source->thread_info.max_latency);
-    }
+    /* Don't allow the per-thread latency to increase. */
 
     /* When we reach this we're officialy fucked! */
 }
--- src/modules/alsa/alsa-sink.c.orig	2012-05-12 22:37:12.641215837 -0500
+++ src/modules/alsa/alsa-sink.c	2012-05-19 09:01:32.743076922 -0500
@@ -293,6 +293,13 @@
 
     if (u->tsched_watermark < u->min_wakeup)
         u->tsched_watermark = u->min_wakeup;
+
+    /* Set the maximum watermark to 50 msec. */
+    size_t max_watermark = pa_usec_to_bytes(50*PA_USEC_PER_MSEC, &u->sink->sample_spec);
+    if (u->tsched_watermark > max_watermark)
+    {
+        u->tsched_watermark = max_watermark;
+    }
 }
 
 static void increase_watermark(struct userdata *u) {
@@ -313,17 +320,7 @@
         return;
     }
 
-    /* Hmm, we cannot increase the watermark any further, hence let's raise the latency */
-    old_min_latency = u->sink->thread_info.min_latency;
-    new_min_latency = PA_MIN(old_min_latency * 2, old_min_latency + TSCHED_WATERMARK_INC_STEP_USEC);
-    new_min_latency = PA_MIN(new_min_latency, u->sink->thread_info.max_latency);
-
-    if (old_min_latency != new_min_latency) {
-        pa_log_info("Increasing minimal latency to %0.2f ms",
-                    (double) new_min_latency / PA_USEC_PER_MSEC);
-
-        pa_sink_set_latency_range_within_thread(u->sink, new_min_latency, u->sink->thread_info.max_latency);
-    }
+    /* Don't allow the per-thread latency to increase. */
 
     /* When we reach this we're officialy fucked! */
 }