text:
AGM Series Hole 1: Calculate the Arithmetic–Geometric Mean
This question was inspired by this HNQ.
About the series
This question is now part of a series about the AGM method. This first post in the series will be about actually calculating the AGM. You may treat this like any other code golf challenge, and answer it without worrying about the series at all. However, there is a leaderboard across all challenges.
What is the Arithmetic–Geometric Mean
The Arithmetic–Geometric Mean of two numbers is defined as the number that repeatedly taking the arithmetic and geometric means converges to. Your task is to find this number after some n iterations.
Clarifications

You take three numbers, a, b, n in any reasonable format.
For n iterations, take the arithmetic and geometric mean of a and b and set those to a and b.
For two numbers a and b, the arithmetic mean is defined as (a + b) / 2.
The geometric mean is defined as √(a * b).
a and b should be approaching each other.
Then, output both a and b.
You don't have to worry about float imprecision and such.
This is code-golf so shortest code in bytes wins!

Test Cases
[0, [24, 6]] -> [24, 6]
[1, [24, 6]] -> [15.0, 12.0]
[2, [24, 6]] -> [13.5, 13.416407864998739]
[5, [24, 6]] -> [13.458171481725616, 13.458171481725616]
[10, [100, 50]] -> [72.83955155234534, 72.83955155234534]

The next one is 1/Gauss's Constant:
[10, [1, 1.41421356237]] -> [1.198140234734168, 1.1981402347341683]

Leaderboard
Stolen from Martin's series.
The following snippet will generate a leaderboard across all challenges of the series.
To make sure that your answers show up, please start every answer with a headline, using the following Markdown template:
# Language Name, N bytes

where N is the size of your submission. If you improve your score, you can keep old scores in the headline, by striking them through. For instance:
# Ruby, <s>104</s> <s>101</s> 96 bytes

/* Configuration */

var QUESTION_IDs = [66068]; // Obtain this from the url
// It will be like http://XYZ.stackexchange.com/questions/QUESTION_ID/... on any question page
var ANSWER_FILTER = "!.FjwQBrX2KXuFkv6p2lChi_RjzM19";

/* App */

var answers = [], page = 1, currentQ = -1;

function answersUrl(index) {
  return "http://api.stackexchange.com/2.2/questions/" +  QUESTION_IDs.join(";") + "/answers?page=" + index + "&pagesize=100&order=desc&sort=creation&site=codegolf&filter=" + ANSWER_FILTER;
}

function getAnswers() {
  $.ajax({
    url: answersUrl(page++),
    method: "get",
    dataType: "jsonp",
    crossDomain: true,
    success: function (data) {
      answers.push.apply(answers, data.items);
      if (data.has_more) getAnswers();
      else process();
    }
  });
}

getAnswers();

var SIZE_REG = /\d+(?=[^\d&]*(?:<(?:s>((?!>).)*<\/s>|((?!>).)+>)[^\d&]*)*$)/;
var NUMBER_REG = /\d+/;
var LANGUAGE_REG = /^#*\s*([^\n,]+)(?=,)/;//

function shouldHaveHeading(a) {
  var pass = false;
  var lines = a.body_markdown.split("\n");
  try {
    pass |= /^#/.test(a.body_markdown);
    pass |= ["-", "="]
              .indexOf(lines[1][0]) > -1;
    pass &= LANGUAGE_REG.test(a.body_markdown);
  } catch (ex) {}
  return pass;
}

function shouldHaveScore(a) {
  var pass = false;
  try {
    pass |= SIZE_REG.test(a.body_markdown.split("\n")[0]);
  } catch (ex) {}
  if (!pass) console.log(a);
  return pass;
}

function getAuthorName(a) {
  return a.owner.display_name;
}

function getAuthorId(a) {
  return a.owner.user_id;
}

function process() {
  answers = answers.filter(shouldHaveScore)
                   .filter(shouldHaveHeading);
  answers.sort(function (a, b) {
    var aB = +(a.body_markdown.split("\n")[0].match(SIZE_REG) || [Infinity])[0],
        bB = +(b.body_markdown.split("\n")[0].match(SIZE_REG) || [Infinity])[0];
    return aB - bB
  });

  var users = {};
  answers.forEach(function (a) {
    var headline = a.body_markdown.split("\n")[0];
    var question = QUESTION_IDs.indexOf(a.question_id);
    var size = parseInt((headline.match(SIZE_REG)||[0])[0]);
    var language = headline.match(LANGUAGE_REG)[1];
    var user = getAuthorName(a);
    var userId = getAuthorId(a);
    if (!users[userId]) users[userId] = {name: user, nAnswer: 0, answers: []};
    if (!users[userId].answers[question]) {
      users[userId].answers[question] = {size: Infinity};
      users[userId].nAnswer++;
    }
    if (users[userId].answers[question].size > size) {
      users[userId].answers[question] = {size: size, link: a.share_link}
    }
  });

  var sortedUsers = [];
  for (var userId in users)
    if (users.hasOwnProperty(userId)) {
      var user = users[userId];
      user.score = 0;
      user.completedAll = true;
      for (var i = 0; i < QUESTION_IDs.length; ++i) {
        if (user.answers[i])
          user.score += user.answers[i].size;
        else
          user.completedAll = false;
      }
      sortedUsers.push(user);
    }

  sortedUsers.sort(function (a, b) {
    if (a.nAnswer > b.nAnswer) return -1;
    if (b.nAnswer > a.nAnswer) return 1;
    return a.score - b.score;
  });

  var place = 1;
  for (var i = 0; i < sortedUsers.length; ++i) {
    var user = sortedUsers[i];
    var row = '
'+ place++ +'.	'+user.name+'	';
    for (var j = 0; j < QUESTION_IDs.length; ++j) {
      var answer = user.answers[j];
      if (answer)
        row += ''+answer.size+'	';
      else
        row += '	';
    }
    row += '	';
    if (user.completedAll)
      row += ''+user.score+'	';
    else
      row += ''+user.score+'	';
    row += '
';
    $("#users").append(row);
  }
}
body { text-align: left !important}

#leaderboard {
  width: 500px;
}

#answer-list {
  padding: 10px;
  width: 290px;
  float: left;
}

#language-list {
  padding: 10px;
  width: 290px;
  float: left;
}

table thead {
  font-weight: bold;
}

table td {
  padding: 5px;
}

td.total {
  font-weight: bold;
  text-align: right;
}

td.missing {
  background: #bbbbbb;
}

<link rel="stylesheet" type="text/css" href="//cdn.sstatic.net/codegolf/all.css?v=83c949450c8b">

Leaderboard

    Missing scores are shown as grey cells. A grey total indicates that the user has not participated in all challenges and is not eligible for the overall victory yet.

	User
        #1
        	Total

{{NAME}}	{{LANGUAGE}}	{{SIZE}}	Link

{{LANGUAGE}}	{{NAME}}	{{SIZE}}	Link

Are the initial numbers positive integers?

"both a or b"—well, which one? Both, or either one?

@Doorknob -_- Its both.

@xnor no. Look at the last test-case.

Making this part of a series causes a kind of unfortunate situation. This is so simple that solutions are all going to look quite similar. And posting similar solutions in languages that were already used is generally frowned upon. I wrote my solution in about 2 minutes, but it's in a language that was already used, and it's the same length. If I follow typical posting etiquette, I won't be able to participate in the series.

the leaderboard snippet seems to be incorrectly reporting Eumel's score.

@JohnE I've edited the answer to get it reported as "21", but I don't think think it should be compared with the byte counts of the other answers in the first place.

CJam, 16 bytes
{{_2$+2/@@*mq}*}

Takes input on the stack as a b n where a and b are doubles. Online demo

Dyalog APL, 22 21 15 bytes
.5∘(+.×,×.*⍨)⍣⎕

Takes (a,b) as right argument, and prompts for n:
(
  +.× dot product of 0.5 and the right argument
 , followed by
  ×.*⍨ "dot power" of the right argument and 0.5*
)⍣⎕ applied numeric-prompt times.
* "dot power" is like dot product, but using multiplication and power instead of plus and multiplication, as follows:
      n
A ×.*⍨ B is ∏ BiA = ∏ B1AB2A
      i=1
-3 bytes thanks to ngn.

Old version:
{((+/÷≢),.5*⍨×/)⍣⍺⊢⍵}

Takes n as left argument and a b as right argument.
⊢⍵ On the RightArg
(...)⍣⍺ recalculate LeftArg times
(+/÷≢) sum divided by tally
, followed by
.5*⍨×/ the square root of the product.
All the test cases:
      f←{((.5×+/),.5*⍨×/)⍣⍺⊢⍵}
      0 1 2 5 10 10 f¨ (24 6)(24 6)(24 6)(24 6)(100 50)(1,2*.5)
┌────┬─────┬────────────────┬───────────────────────┬───────────────────────┬───────────────────────┐
│24 6│15 12│13.5 13.41640786│13.45817148 13.45817148│72.83955155 72.83955155│1.198140235 1.198140235│
└────┴─────┴────────────────┴───────────────────────┴───────────────────────┴───────────────────────┘

Is f⍣⍺⊢⍵ or similar an idiom you use professionally?

@ThomasKwa Yes, see e.g. Of⍣core⊢TREE at miserver.dyalog.com (click the big "D" and scroll to line [266]).

TI-BASIC, 22 bytes
Input N
For(I,1,N
{mean(Ans),√(prod(Ans
End
Ans

Does exactly what the algorithm says. Takes N from the prompt, and A and B through Ans as a two-element list.
If N is 0, the For( loop is skipped entirely.

JavaScript ES7, 48 43 bytes
-5 thanks to Downgoat!
f=(n,a,b)=>n?f(n-1,(a+b)/2,(a*b)**.5):[a,b]

Very simple recursive function.

(a*b)**.5 is shorter than Math.sqrt(a*b). example

@Downgoat That's ES7, but meh.

MATLAB/Octave, 69 65 bytes
function [a,b]=r(a,b,n)
for i=1:n;j=(a+b)/2;b=(a*b)^.5;a=j;end

You could do b=(a*b).^5 directly as you are not reusing b again in that iteration and save 4 bytes.

Jelly, 9 bytes
SH;P½¥ðṛ¡

Try it online!
How it works
SH;P½¥ðṛ¡    Input: x (vector) -- y (repetitions)

SH           Take the sum (S) of x and halve (H) the result.
   P½        Take the product (P) of x and the square root (½) of the result.
     ¥       Combine the last two instructions in a dyadic chain.
  ;          Concatenate the results to the left and to the right.
      ð      Push the preceding, variadic chain; begin a new, dyadic chain.
       ṛ     Return the right argument (y).
        ¡    Repeat the pushed chain y times.

C++, 108 102 100 bytes
Thank you to @RetoKoradi and @AlexA for saving me 6 bytes.
This is non-competitive, because C++ is not a good golfing language. Did this for fun :)
#include<cmath>
std::string f(float a,float b,int n){return n==0?a+" "+b:f((a+b)/2,sqrt(a*b),n-1);}

This is a simple recursion function, very similar to the JS answer.

You can get rid of the spaces after the commas. Also, using float instead of double is shorter.

You can also remove the space in the #include line.

Wow, I'm stupid not to notice that. Thanks!

I would consider f(float*s) which takes a pointer to 3 floats to be in a "reasonable format". Not sure if that actually makes it shorter.

K5, 15 bytes
Very literal:
{(+/x%2;%*/x)}/

In action:
 {(+/x%2;%*/x)}/[0; 24 6]
24 6
 {(+/x%2;%*/x)}/[5; 24 6]
1.345817e1 1.345817e1

Unfortunately, this does not work in oK because that interpreter does not currently support projection (currying) of adverbs. Works in the real k5.
In oK, it would currently be necessary to wrap the definition in a lambda:
  {x{(+/x%2;%*/x)}/y}[5; 24 6]
13.4582 13.4582

J,  18 13 bytes
-:@+/,%:@*/^:

Usage:
   agm =: -:@+/,%:@*/^:
   5 agm 24 6
13.4582 13.4582

Wow, this works. Conjunctions are weird. I would expect this expression to be an adverb (which it can be), but if presented with arguments it is also a function.

Seriously, 11 bytes
,p`;π√@æk`n

Hex Dump:
2c70603be3fb40916b606e

Try it online
Explanation:
,                    Read in the list as [n,a,b]
 p                   pop list to yield: n [a,b]
  `      `n          Push a quoted function and run it n times.
   ;                 Duplicate [a,b] pair
    π√               Compute its product and square root it (GM)
      @              Swap the other copy of the pair to the top
       æ             Compute its mean.
        k            Compile the stack back into a list.

Japt, 24 bytes 25 33
Saved 9 7 bytes thank to @ETHproductions
Uo r@[VW]=[V+W /2(V*W q]

Takes advantage of ES6 destructuring.
Try it online
Ungolfed && Explanation
Uo r@[VW]=[V+W /2(V*W q]

       // Implicit: U: 1st input, V: 2nd input, W: 3rd input
Uo     // Range from 0 to 1st input
r@     // Loop over range
  [V,W]=    // Set 2nd and 3rd input to...
   [V+W /2,   // Add 2nd and 3rd inputs, divide by 2
   (V*W q]    // Multiple 2nd and 3rd inputs, find square root
            // Set's to the above respectively
       // Implicit: return [V,W]

Uo generates a range of numbers from 0 to U, so Uo m@[V,W]=[V+W /2,(V*W q] should work. (Untested)

Oh, and you shouldn't need the commas at all. :)

@ETHproductions thanks yet again! :)

Oh dear, this fails for any U other than 1, outputting each loop as it goes. Here's one that works properly: Uo £[VW]=[V+W /2(V*W q]};[VW]

@ETHproductions thanks, but using r seemed to work also

Ah, r works much better; nice job outsmarting me in my own language. ;)

Pyth, 12
u,.OG@*FG2EQ

Test Suite
Explanation
u,.OG@*FG2EQ    ##  implicit: Q = eval(input())
u         EQ    ##  reduce eval(input()) times, starting with Q
                ##  the reduce lambda has G as the previous value and H as the next
  .OG           ##  arithmetic mean of last pair
     @*FG2      ##  geometric mean of last pair, uses *F to get the product of the list
                ##  and @...2 to get the square root of that
 ,              ##  join the two means into a two element list

Forgot about @ and .O, but I didn't even know the new purpose of E.

@orlp ah, didn't see your post, my bad I would have just suggested these in the comments. And yeah, keeping track of all the changing stuff is a bit of a struggle :P

Matlab, 54 bytes
function x=f(x,n)
for k=1:n
x=[mean(x) prod(x)^.5];end

Example:
>> f([24 6], 2)
ans =
  13.500000000000000  13.416407864998739

Minkolang v0.14, 23 bytes
Try it here!
$n[$d+2$:r*1Mi2%?!r]$N.
$n                      C get all input C
  [                ]    C pop N; repeat inner N times C
   $d                   C duplicate stack [1,2] => [1,2,1,2] C
     +                  C add top two elements C
      2$:               C divide by two C
         r              C reverse stack (get the other two) C
          *             C multiply them together C
           1M           C take square root C
             i2%?!r     C reverse the stack if an odd step number C
                    $N  C output stack
           1M           C take square root C
             i          C get step in for loop C

Python 3, 65 55 bytes
Thanks to mathmandan for pointing out a shorter version using the lambda operator.
f=lambda a,b,n:f((a+b)/2,(a*b)**.5,n-1)if n else(a,b)

My original version:
def f(a,b,n):
 if n:f((a+b)/2,(a*b)**.5,n-1)
 else:print(a,b)

To my chagrin, a recursive function (a la the JavaScript and C++ answers) was shorter than a simple for loop.

You can shorten this a little with lambda and the ternary if/else operator: f=lambda a,b,n:f((a+b)/2,(a*b)**.5,n-1)if n else(a,b)

No problem!  (Also, I think this is 53 bytes.)

The .py file I have saved is listed as being 55 bytes.  Is there a better way to calculate program size?

Sometimes people on this site copy and paste their code into https://mothereff.in/byte-counter .  If you're wondering about the discrepancy, I would guess that Windows is inserting an unnecessary newline character at the end of your .py file (and Windows counts a newline as 2 bytes instead of 1).  Either way, you don't have to count that last newline as part of your code for scoring purposes. If you do post a multiple-line entry, you should count 1 for each newline character, not 2, and not including any newline at the end of your last line of code. (As far as I understand the rules anyway!)

Pyth, 15 bytes
u,^*FG.5csG2vzQ

Mathematica, 31 30 bytes
Saved one byte thanks to Martin Büttner.
{+##/2,(1##)^.5}&@@#&~Nest~##&

Usage:
In[1]:= {+##/2,(1##)^.5}&@@#&~Nest~##&[{24, 6}, 5]

Out[1]= {13.4582, 13.4582}

R, 66 bytes
f=function(a,b,n){while(n){x=(a+b)/2;b=(a*b)^.5;n=n-1;a=x};c(a,b)}

Usage:
> f(24,6,0)
[1] 24  6
> f(24,6,1)
[1] 15 12
> f(24,6,2)
[1] 13.50000 13.41641
> f(24,6,3)
[1] 13.45820 13.45814
> f(24,6,4)
[1] 13.45817 13.45817
> f(100,50,10)
[1] 72.83955 72.83955
> f(1,1.41421356237,10)
[1] 1.19814 1.19814

You can remove the function name to save 2 bytes.

Lua, 62 bytes
n,a,b=...for i=1,n do a,b=(a+b)/2,math.sqrt(a*b)end print(a,b)

Uses command line arguments from ... to assign to n, a and b, a nifty trick I learned about Lua recently.

Python 2, 62 61 62 bytes
def f(a,b,n):
 while n:a,b=(a+b)/2.,(a*b)**.5;n-=1
 print a,b

The program should only print once, when it terminates.

My misunderstanding.  Fixed.

Haskell, 47 Bytes
f a b 0=(a,b)
f a b n=f((a+b)/2)(sqrt$a*b)(n-1)

you could save some bytes taking a b as a couple in f:   f x 0=x;f(a,b)n=f((a+b)/2,sqrt$a*b)$n-1

And define the function infix.

Haskell, 40 bytes
(!!).iterate(\(a,b)->((a+b)/2,sqrt$a*b))

An anonymous function. Example usage:
>> let f=(!!).iterate(\(a,b)->((a+b)/2,sqrt$a*b)) in f (1.0,1.41421356237) 10
(1.198140234734168,1.1981402347341683)

The lambda function (\(a,b)->((a+b)/2,sqrt$a*b)) takes the arithmetic and geometric mean on a tuple. This is iterated starting with the first input (a tuple), and then (!!) indexes the second input to specify the number of iterations.

Perl, 60 bytes
perl -ape'F=($F[0]/2+$F[1]/2,sqrt$F[0]*$F[1])for 1..shift@F;$_="@F"'

N.B.: Per this meta post, I believe I've got the scoring correct. The actual code (between single quotes) is 58 characters, then I added +2 for a and p flags as that's the difference from the shortest invocation, perl -e'...'
Vague complaints
I have this nagging feeling I'm missing an obvious improvement. I know, "welcome to code golf", but I mean more than usual I believe there's an easy opportunity to shorten this.
Early on, I had messed around with using $\ as the second term with some success, but the above approach ended up being 2 bytes shorter, even with the extra ap flags required. Similarly, avoiding the explicit $_ assignment would be nice, but the loop makes that difficult.
The shift@F bugs me, too; if I don't do it that way, though (or use @F=(0,...,...) instead, which doesn't save any bytes), there's an off-by-one error with the @F assignment.
Example
echo 5 24 6 | perl -ape'F=($F[0]/2+$F[1]/2,sqrt$F[0]*$F[1])for 1..shift@F;$_="@F"'

Outputs
13.4581714817256 13.4581714817256

Julia, 49 bytes
(a,b,n)->(for i=1:n;a,b=(a+b)/2,√(a*b)end;(a,b))

Pretty direct iterative algorithm. Using the√ symbol and the multiple return saves a few bytes, but the for loop syntax costs a few.

Julia, 42 bytes
f(a,b,n)=n>0?f((a+b)/2,(a*b)^.5,n-1):(a,b)

This is a recursive function f that accepts three numbers and returns a tuple.
Ungolfed:
function f(a::Real, b::Real, n::Integer)
    if n > 0
        # Recurse on the arithmetic and geometric means, decrementing n
        return f((a + b) / 2, sqrt(a * b), n - 1)
    else
        # Return the pair
        return (a, b)
    end
end

LabVIEW, 21 LabVIEW Primitives
Primitives counted as per this meta post.

pretty staightforward not much to explain.

CJam, 16 bytes
{{_:+2/\:*mq]}*}

This is an anonymous function. The input is a list with the two values (as doubles), followed by the iteration count. Try it online with I/O code for testing.
I wouldn't normally have posted this because @PeterTaylor posted an equally long CJam answer before I saw the question. But since this is advertised as the start of a series, I wanted to keep my options open in case the series is interesting.
While the length is the same as Peter's answer, the code is not. I chose a different input format by taking the two values in a list, where Peter used separate values. So while there's not much to it with either input format, the code looks quite different.
{     Start loop over number of iterations.
  _     Copy the current pair of values.
  :+    Reduce pair with + operator.
  2/    Divide by 2.
  \     Swap second copy of pair to top.
  :*    Reduce pair with * operator.
  mq    Calculate square root.
  ]     Wrap the two new values in a list for next iteration.
}*    End iteration loop.

><> (Fish), 66 bytes
Uses newtons method to approximate the square root in 15 steps
iii>:?v~naon;
*&f:?v\1-@:&$:@$:@+2,@
$@:$@//,2+,@:
4{~$~/\@@1-21.0

Try it

Scala, 58 bytes
Golfed version. Try it online!
(n,a,b)=>if(n>0)f(n-1,(a+b)/2,Math.sqrt(a*b))else s"$a,$b"

Thunno 2, 6 bytes
{[m;pƭ

Try it online!
Takes input on the stack. This default was +11/-5 at the time of posting (Wayback Machine).
Explanation
{[m;pƭ  # Implicit input
{       # First input no. times:
 [ ;    #  Parallelly apply:
  m     #   Arithmetic mean
    pƭ  #   Square root of product
        # Implicit output

Go, 95 92 bytes
import(."math";."fmt")
func f(n,a,b float64){for;n>0.;n--{a,b=(a+b)/2,Sqrt(a*b)}
Print(a,b)}

Attempt This Online!
Prints to STDOUT, with a space separating the means.
The explicitly looped solution is the same length as the recursive solution, funnily enough (all byte savings from making it recursive is lost by needing an extra else{} block to prevent output when n>0).

-3 by @The Thonnu

92 bytes

Perl 6,  53  47 bytes
{(($^a,$^b),->(\a,\b){((a+b)/2,sqrt(a*b))}...*)[$^n]} # 53 bytes

usage:
# give it a name
my &code = {(($^a,$^b),->(\a,\b){((a+b)/2,sqrt(a*b))}...*)[$^n]}

say code 100,50,10;          # (72.8395515523453 72.8395515523453)
say code 1,1.41421356237,10; # (1.19814023473417 1.19814023473417)

If I change the input from a,b,n to (a,b),n I can save a few bytes.
{($^l,->(\a,\b){((a+b)/2,sqrt(a*b))}...*)[$^n]} # 47 bytes

usage:
my &code = {($^l,->(\a,\b){((a+b)/2,sqrt(a*b))}...*)[$^n]}

say code (100,50),10;          # (72.8395515523453 72.8395515523453)
say code (1,1.41421356237),10; # (1.19814023473417 1.19814023473417)

say code (24,6),$_ for 0,1,2,5;
# (24 6)
# (15 12)
# (13.5 13.4164078649987)
# (13.4581714817256 13.4581714817256)

{
  (
    $^l,          # first 2 element tuple
    ->            # pointy block (lambda)
      (\a,\b)     # take a single tuple, and give its 2 elements each a name
    {
      (           # create a 2 element tuple
        (a+b)/2,  # arithmetic mean
        sqrt(a*b) # geometric mean
      )
    } ... *       # create a lazy infinite sequence of tuples
  )[ $^n ]        # take the nth "tuple" from the outer sequence
}

Really I would swap out the ... * with ... -> (\a,\b) { a =~= b }, then there would be no need for the $^n parameter.
( do not use == instead of =~=, or it may not stop )
my &code = {($^l,->(\a,\b){((a+b)/2,sqrt(a*b))}...->(\a,\b){a=~=b})[*-1]}

say code (24,6);           # (13.4581714817256 13.4581714817256)
say code (100,50);         # (72.8395515523453 72.8395515523453)
say code (1,1.41421356237) # (1.19814023473417 1.19814023473417)

Prolog, 80 bytes
Code:
p(A,B,0):-write([A,B]).
p(A,B,N):-X is(A+B)/2,Y is sqrt(A*B),M is N-1,p(X,Y,M).

Example:
p(100,50,10).
[72.83955155234534, 72.83955155234534]

Try it online here

Java, 103 96 84 bytes
String f(int n,double a,double b){return n>0?f(n-1,(a+b)/2,Math.sqrt(a*b)):a+","+b;}

Verify all testcases.
Old version (96 bytes):
String f(int n,double a,double b){for(;n>0;a=(a+b)/2,b=Math.sqrt((b-2*a)*b))n--;return a+","+b;}

Old version (103 bytes):
String f(int n,double a,double b){double t;for(;n>0;t=(a+b)/2,b=Math.sqrt(a*b),a=t)n--;return a+","+b;}

Vyxal, 6 bytes
(₌ṁΠ√W

Try it Online!
The obvious 6 byte solution
Explained
(₌ṁΠ√W
(       # n times:
 ₌ṁ     #   Push the mean of the top of the stack
 ₌ Π    #   And the product of the top of the stack
    √   #   Square root that product
     W  #   And wrap into a list

Created with the help of Luminespire.


text:
Find the follow sets
The challenge below requires you to be familiar with formal parser theory. If you don't know what the question is asking because you don't know what the terms mean, context-free grammars and first/follow sets are covered in a lot of university courses.
I can recommend this Stanford course, in particular handouts 08 and 09 (from page 7). I've extracted also extracted a cheat sheet from these handouts  - I recommend anyone attempting this challenge to read it.

Write a program or function that given a context-free grammar finds the follow set of every nonterminal. Informally, the follow set of a nonterminal is a set of terminals and $ (meaning end-of-input) that you can possibly find after that terminal in a valid sentence.
The input is given as a single printable ASCII string or array of printable ASCII lines. You may output the sets in any reasonable format, using $ (either as literal output, or string inside a set, etc) to indicate the end of input. You may assume that the input is always valid according to the format below.
The context free grammar is given in a very simplified manner. Every line contains a single production. Every production is a space separated list of symbols. A terminal is a string of characters surrounded by apostrophes (e.g. '**'). For simplicity you may assume that terminals do not contain spaces, but it would be nice if your program allows it. A nonterminal may be any string not containing spaces or $. The empty production (normally indicated with ε) is simply a line containing only the left hand side nonterminal. The first line is the production defining the start symbol.
As an example, the following grammar:
S → aSa | bSb | ε
Will be given as:
S 'a' S 'a'
S 'b' S 'b'
S

Example input / outputs:
In:
S 'a' S 'a'
S 'b' S 'b'
S

Out:
S {'a', 'b', $}

In:
S A B C
A 'a'
A C 'b'
A
B C
B 'd' A
B
C 'e'
C 'f'

Out:
S {$}
A {'d', 'e', 'f'}
B {'e', 'f'}
C {'b', 'e', 'f', $}

In:
Start Alice Bob
Alice Charlie 'a'
Alice
Bob Bob 'a' Alice Charlie
Bob '!!!'
Charlie 'b'
Charlie

Out:
Start {$}
Alice {'a', '!!!', 'b', $}
Bob {'a', $}
Charlie {'a', $}

Shortest code in bytes wins.

Assuming that people know what a context free grammar is seems fine, but I think it wouldn't hurt the challenge if you included the definition of a follow set right here instead of just linking to it.

This brings back some memories from "Compiler Construction" at university, where we had to solve lots of similar tasks.

Perl, 257 bytes
Includes +4 for -0p
Give grammar on STDIN (without trailing spaces. Make sure to remove the extra space in the second example). Assumes non-terminal names only contain letters, digits and _. Uses # instead of $ to indicate end of input. Can handle literals containing spaces
perl -M5.010 follow.pl
E T e
e '+' T e
e
T F t
t '*' F t
t
F '(' E ')'
F 'id'
^D

Outputs the follow sets as a list of non-terminal literal in no particular order. For the example above it outputs:
F ')'
F #
t ')'
t #
T ')'
T #
F '+'
t '+'
T '+'
F '*'
e ')'
e #
E ')'
E #

follow.pl:
#!/usr/bin/perl -0n
s/'.*?'/~$&/eg;s% (?=(\w.*\n))%$_.=">$1"%reg;/\s/;$_.=">$` #\n";s%^((\w+)\K ?\S*).*%$s{$1}++||"\$a.=s/ $2\\b/$&/rg"%eemgr,s%^(\w+ ).*?(\w+)$%"\$a.=s/>$1/>$2 /rg"%eermg,$_.=$a,s%>.*\xd8\K .*%%g,s%.+\n%$&x!/\n$&/g%eg until$$_++;s/\xd8.*?\xd8/~$&/eg;say/>(\w+ \W\S*\n)/g

Works as shown, but replace \xd8 and \n by their literal versions to get the claimed score.
It should be possible to improve this since converting the first sets to the follow sets is currently very awkward.

Python3, 271 bytes
def f(g):
 d={g[0][0]:{''}}
 q=[(i[1:],[])for i in g]
 for a,S in q:
  for i,A in enumerate(a):
   if A and"'"!=A[0]:
    if a[i+1:]and"'"==a[i+1][0]:d[A]={*d.get(A,[]),a[i+1],''}
    for j,*k in g:
     if j==A and(j,k)not in S:q+=[(a[:i]+k+a[i+1:],S+[(j,k)])]
 return d

Try it online!

The solution is not complete yet. It's missing the end-of-input / $ symbol output.

source: Stack Exchange

license: Creative Commons - Attribution Share-Alike - https://creativecommons.org/licenses/by-sa/3.0/

all_licenses: Creative Commons - Attribution Share-Alike - https://creativecommons.org/licenses/by-sa/4.0/

sort: votes


text:
Quat. Quine + Cat
A Quat is a combination of a quine and the popular esolang cat program.
Challenge
The challenge is to write a standard cat program. Whatever the user inputs, the program will echo the input to stdout.
However, when the length of the input is greater than 0 and a multiple of 4, the program should output its own source code. Quat comes from the Portuguese quatro, which translates to 'four'.
Rules

Standard loopholes apply
You may not read the source code from a file
An empty input should produce an empty output

Bonus
If your input length is a multiple of 4 you can earn a 25% bonus by printing the quine length/4 times. Another 5% bonus if you seperate the output by spaces (no trailing space allowed).
Test cases
The following test cases apply for the program in%4=0?cat:self (not a real language).
<empty input> -> <empty output>
input -> input
1234 -> in%4=0?cat:self
12345678 -> in%4=0?cat:self 0% bonus
12345678 -> in%4=0?cat:selfin%4=0?cat:self 25% bonus
12345678 -> in%4=0?cat:self in%4=0?cat:self 30% bonus
Scoring
This is code-golf. Shortest code in bytes wins.

Can the input have multiple lines?

@LegionMammal978 Ofcourse, let \n be the new line character, for 1 byte of input

Okay, it just complicates input in my language, which must be read one line at a time.

Are built-ins for quines allowed? (see: Seriously)

@FlagAsSpam I don't mind, but I think a meta-post is more appropriate here.

@Bas I meant for your question, not in general. But okie den.

@FlagAsSpam The OP of What counts as a proper quine? says "Clearly, any quine in HQ9+ or a related language is not a proper quine," and I don't see any contention over that point in the answers. I'm pretty sure I've seen that same point made in other posts.

Is there any maximum input size we have to support? (I.e. do we have to buffer all 5 gigabytes of input until deciding whether to output them or the own source code?)

CJam, 23 * 0.75 = 17.25 bytes
Standard generalised quine...
{`"_~"+q:Q,4md@@*Q\?}_~

or
{`"_~"+q_,4md@@]:\*?}_~

Test it here.
Explanation
{`"_~"+  e# Generalised quine framework. Leaves the source code on the stack.
  q:Q    e# Read input and store it in Q.
  ,      e# Get its length.
  4md    e# Divmod 4.
  @      e# Pull up the source code.
  @      e# Pull up the div.
  *      e# Repeat the source code that many times.
  Q\     e# Push the input and swap it below the repeated source.
  ?      e# Pick the right output based on the modulo.
}_~

The other version avoids the use of a variable by using the stack-rotation trick ]:\.

Pyth, 33 * .75 = 24.75
?%lz4z*/lz4jN*2]"?%lz4z*/lz4jN*2]

Test Suite
Standard Pyth quine using join. This is only a true quine on the online interpreter, which doesn't add a final trailing newline.
Getting the final bonus reults in a score of 39 * .7 = 27.3:
?%lz4zjd*/lz4]jN*2]"?%lz4zjd*/lz4]jN*2]

Emacs Lisp (323 * 0.75 = 242.25)
((lambda (s) (let* ((a (read-string "")) (l (string-bytes a))) (if (> (% l 4) 0) (message a) (dotimes (v (/ l 4)) (prin1 (list s (list (quote quote) s))))))) (quote (lambda (s) (let* ((a (read-string "")) (l (string-bytes a))) (if (> (% l 4) 0) (message a) (dotimes (v (/ l 4)) (prin1 (list s (list (quote quote) s)))))))))

This uses Lisp's quoting mechanic to give the source code as input to itself.
Old cheating version
:; exec emacs -Q -script $0
(find-file(nth 2 command-line-args))(set'b(buffer-string))(set's(read-string""))(set'l(string-bytes s))(if(>(% l 4)0)(message s)(dotimes(v(/ l 4))(message"%s"b)))

Ungolfed:
:; exec emacs -Q -script $0
(find-file(nth 2 command-line-args)) ; open self
(set'b(buffer-string))               ; read own code to string
(set's(read-string""))               ; read input
(set'l(string-bytes s))              ; length of input
(if(>(% l 4)0)                       ; l % 4 > 0 ?
    (message s)                      ; output input
  (dotimes(v(/ l 4))                 ; (implicit else) repeat l/4 times
    (message"%s"b)))                 ; output own code

Doesn't this violate the rule that "You may not read the source code from a file"?

@ThisSuitIsBlackNot you're right... guess I'll have to look for a non-cheating Quine in Elisp

Seriously, 8 9 bytes
Q,ó;l4@%I

Try it Online
(Hit enter once in the input box to test empty input.)
The first bonus can be done in 12 bytes (16*.75):
Q,ó;l;4@\(*)4@%I

Explanation:
Q                   Push program source to stack
 ,ó                 Push input string, and terminate if it's empty
   ;l               Push length of input
     4@%            Take the length mod 4.
        I           Pick the next stack element (input) if nonzero,
                    else the next next (program source)

Since some people don't like the use of Seriously's quining built-in, I provide this 22 byte version that doesn't use Q for reference:
`è";ƒ"(+,ó;l4@%I`;ƒ

If you are one of those people, consider this the definitive version (for now) and then go start a meta thread about the use of built-ins in quines.

In a [tag:quine] challenge, I believe you cannot read the program source.

The instructions say you cannot read it from a file. Q is a built-in command that pushes the source to the stack when it is empty. I think this falls under "playing to the language's strengths"

I think a built-in quining function does read the source code. That's like using a built-in for a binomial coefficient challenge, when the challenge says "no built-in factorials".

It doesn't say "don't use quining built-ins"... And not using built-ins is not an accepted standard loophole

Q does not read the program's source from a file, technically - it's in RAM, stored by the interpreter. Since the rule specifically states "from a file", this should be valid.

I don't really understand why this is more popular than any previous languages with built-in quine operators, including HQ9+ and Ismael's MarioGolf. But in any case your program doesn't seem to work correctly for empty input.

Hmm. I think when I tested empty input in the REPL, it actually gave it a newline, so it looked empty... Or something like that.

@MartinBüttner Seriously is a valid programming language by our definition, unlike HQ9+. I don't know why Mario Golf isn't more popular; I think it's neat. The main difference I can see is that Q isn't just a built-in one-byte quine; it's also the simplest way to do recursion in functions.

Vitsy, 18 17 bytes
So close. Yus. I am now winning amongst non-built in quiners! glares at Seriously
zl4M([&'rd3*8\}]Z
z                 Grab ALL THE INPUT! :D
 l4M([         ]  If the input is a multiple of four, do the stuff in brackets.
      &           Generate a new stack and move to it.
       'rd3*      Standard quine.
            8\}   Push the bottom 8 items of the stack to the top.
                Z Output the current stack.
There's no reason for me to go after the bonuses - they'd chuck on a lot more bytes.
Cheating quine version, 12 bytes:
zl4M([&iG`]Z
zl4M([&   ]Z  Same as above.
       i      Push -1.
        G     Get the name of the file with this index of use (-1 is self)
         `    Read the file with the given name and push its contents to the stack.

G`  is cool! It seems more legit than a simple Q command.

Seems less legit to me, seeing as how it requires file I/O.

@CᴏɴᴏʀO'Bʀɪᴇɴ Quintopia is right - just because I get the file reference doesn't mean it's any better. :P

Perl, 68 65 * 0.75 = 48.75 bytes
perl -e'$_=q{print+($l=($~=<>)=~y///c)%4?$~:"\$_=q{$_};eval"x($l/4)};eval'

See the online test suite here.
Broken down
perl -e'
    $_=q{                      # store source code in $_
        print+(
            $l=($~=<>)=~ y///c # read STDIN into $~, assign length to $l
        )%4 ?                  # if length is a multiple of 4
             $~ :              # print $~
             "\$_=q{$_};eval"  # otherwise, print source code
             x($l/4)           # length/4 times
    };
    eval'                      # eval $_ to execute its contents

JavaScript, 57 56 72 bytes * 0.75 = 54
Thanks to @Neil for a one byte savings!
(f=_=>alert(!(p=prompt())||(l=p.length)%4?p:`(f=${f})()`.repeat(l/4)))()

The shortest solution I could find was pretty straight-forward.
So, here's a couple bonus (more interesting) solutions:
JavaScript, 82 81 bytes * 0.75 = 60.75
f=_=>{try{p=prompt();a=`f=${f};f()`.repeat(p.length/4)}catch(e){a=p}alert(a)};f()

This one abuses repeat's functionality of throwing an exception if passed a non-integer.

JavaScript, 83 bytes * 0.70 = 58.1
(f=_=>alert((a=(p=prompt()).split(/.{4}/)).pop()?p:a.fill(`(f=${f})()`).join` `))()

This last one is definitely my favorite, splitting the input on every four characters using the regex /.{4}/. If there are any characters left at the end of the string when we pop, it isn't divisible by 4, so alert the input. Otherwise, the pop reduced the array's length by one, so at this point the array's length is equal to the input length / 4. In this case, just fill it with the quine and join with spaces.

I think you can save a byte by inverting the condition; change the && to ||, put a ! before the (p=prompt()), remove the <1 and move the p to be on the inside of the ?:.

Python 3.8 (pre-release), 71 bytes -30% = 49.7
exec(a:="s=len(t:=input());print(*s%4and[t]or s//4*['exec(a:=%r)'%a])")

Try it online!

Mathematica, 229 bytes
($RecursionLimit = Infinity; WriteString[$Output, If[Mod[StringLength[a = (If[(a = InputString[]) === EndOfFile, "", StringJoin["\n", a, #0[]]] & )[]], 4] == 1, ToString[#0, InputForm][], If[a == "", "", StringDrop[a, 1]]]]) & []

All of the whitespace is for the InputForm of the program to match its actual code.

Javascript ES6, 45 bytes
$=(_=prompt())=>_.length%4?_:`$=${$};$()`;$()

Extension of my 21-byte Bling Quine. Hope that mixing prompt and function output is allowed.

JavaScript, 33 bytes
f=(i,l=i.length)=>l%4?i:("f="+f).repeat(l/4)

+44 bytes
-25% bonus

Other solutions:
44 36 bytes

f=(i,l=i.length)=>l%4?i:("f="+f).repeat(!!l)

f=(i,l=i.length)=>l%4?i:l?("f="+f):i

38.5 bytes
f=(i,l=i.length)=>l%4?i:Array(l/4).fill("f="+f).join` `

+55 bytes
-25% bonus
-5% bonus

Zsh, 78 bytes
s='s=\47%s\47
0()printf $s $s
$[$#1&3]||<<<$1'
0()printf $s $s
$[$#1&3]||<<<$1

Try it online!
Uses &3 instead of %4 so the % doesn't get interpreted by printf.
The bonus could be added with repeat $[$#1/4]  before the printf in both places, but unfortunately it gives a higher score (\$ 75\% \times 110 = 82.5 \$).

Vyxal, 38 - 25% = 28.5 bytes
`?DL4%[|[$:q$+$L4/*`?DL4%[|[$:q$+$L4/*

Try it Online!
`...`                      # Source code in string
     ?                     # Grab input
      D                    # Triple
       L4%                 # If length is (not) divisible by 4
          [|               # Do nothing, otherwise
            L0>[           # If length is positive
                $          # Swap (to get source-code string)
                 :q$+      # Standard quine - duplicate, uneval, swap, concatenate
                     $     # Swap to get original input
                      L4/  # Length / 4
                         * # (Source code) that many times

Trying to get the bonus is sadly longer.

Excel, 204 bytes - 30% = 142.8 score
=LET(a,LEN(A1),c,CHAR(34),q,"=LET(a,LEN(A1),c,CHAR(34),q,#,IF(MOD(a,4)=0,TRIM(REPT(SUBSTITUTE(q,CHAR(35),c&q&c),a/4)),LEFT(A1,1))) ",IF(MOD(a,4)=0,TRIM(REPT(SUBSTITUTE(q,CHAR(35),c&q&c),a/4)),LEFT(A1,1)))

Link to Spreadsheet
2nd Place, 191 bytes - 25% = 143.25 score
=LET(a,LEN(A1),c,CHAR(34),q,"=LET(a,LEN(A1),c,CHAR(34),q,#,IF(MOD(a,4)=0,REPT(SUBSTITUTE(q,CHAR(35),c&q&c),a/4),LEFT(A1,1)))",IF(MOD(a,4)=0,REPT(SUBSTITUTE(q,CHAR(35),c&q&c),a/4),LEFT(A1,1)))

3rd Place, 179 bytes
=LET(a,LEN(A1),c,CHAR(34),q,"=LET(a,LEN(A1),c,CHAR(34),q,#,IF(a*(MOD(a,4)=0),SUBSTITUTE(q,CHAR(35),c&q&c),left(A1,1)))",IF(a*(MOD(a,4)=0),SUBSTITUTE(q,CHAR(35),c&q&c),LEFT(A1,1)))

Best Cheesy Answer, 59 byte - 25% = 44.25 score
=LET(a,LEN(A1),IF(MOD(a,4)=0,REPT(FORMULATEXT(B8),a/4),A1))

all_licenses: Creative Commons - Attribution Share-Alike - https://creativecommons.org/licenses/by-sa/3.0/


text:
Remove more than n consecutive vowels from input string
I don't like strings with more than three vowels in a row. Can you write a program that removes all the vowels I don't want from words?
You may write a program or function, taking input via STDIN (or closest alternative), command-line argument or function argument and outputting the result via STDOUT (or closest alternative), function return value or function (out) parameter.
Input is a string containing only printable ASCII character (0x20 to 0x7E, inclusive).
Output is a string containing only runs of at most 3 consecutive vowels. If there is a run of more than 3 consecutive vowels in the input string, your program should produce an output string including the first three vowels encountered in that run, discarding any further consecutive vowels.
Y is not a vowel for the purposes of this challenge.
This is code golf, so the shortest code (in bytes) wins.
Test Cases
"Aeiou" => "Aei"
"screeeen" => "screeen"
"We're queueing up for the Hawaiian movie." => "We're queung up for the Hawaiin movie."
"Spaces break runs: aei iou." => "Spaces break runs: aei iou."

You should include some more tests with mixed cases, like aaYYAAaaaAERGH.

Pyth, 21 bytes
sfg3=Z&}rT0"aeiou"hZz

Try it online: Demonstration or Test Suite
Explanation:
I iterate through all chars and keep track of how many vowels I passed using a counter. Every time I pass a char, which is not a vowel, I reset the counter to 0. I reomve chars, whenever the counter is > 4.
sfg3=Z&}rT0"aeiou"hZz   implicit: z = input string
                                  Z = 0
 f                  z   test every char T in z; keep chars, that return true:
        rT0                convert T to lower
       }   "aeiou"         test if T is a vowel
      &           hZ       logical and with Z+1,
                           gives 0 if ^ is false, otherwise Z+1
    =Z                     update Z with this value
  g3                       test if 3 >= Z
s                       sum up all remaining chars and print

Unreadable, 1647 bytes

'"""""'""'""""""'"""'""""""'""'""'"""'""""""""""'"""""""""'"""""""""'""""""'"""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'"""""""'"""'"""""""""'""""""'"""'""""""""'""""""""'""""""""'""""""""'"""""""'"""'"""""""""'""""""'"""'""""""""'""""""""'""""""""'""""""""'"""""""'"""'"""""""""'""""""'"""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'"""""""'"""'"""""""""'""""""'"""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'"""""""'"""'"""""""""'""""""'"""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'"""""""'"""'"""""""""'""""""'"""'""""""""'""""""""'""""""""'""""""""'"""""""'"""'"""""""""'""""""'"""'""""""""'""""""""'""""""""'""""""""'"""""""'"""'"""""""""'""""""'"""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'"""""""'"""'"""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'""""""""'"""""""'"""'""""""""'"""'"""'"""'"""'"""'"""'"""'"""'"""'"""'"""'"""""""""'""'""'""'""'""""""'""'"""'""""""""'"""""""'""'"""'"'"""""""'""'""'"""'""""""'""'"""'""'"""""""'""'"""'""""'"'"""""""'""'""'"""'""""""'""'"""'""""""""'"""

Explanation
This program is equivalent to pseudocode like this:
while (cp = (ch = read)) + 1 {
    (
        (cp -= 65) ?    // A
            (cp -= 4) ?     // E
                (cp -= 4) ?     // I
                    (cp -= 6) ?     // O
                        (cp -= 6) ?     // U
                            (cp -= 12) ?    // a
                                (cp -= 4) ?     // e
                                    (cp -= 4) ?     // i
                                        (cp -= 6) ?     // o
                                            (cp - 6) ?      // u
                                                0
                                            : 1
                                        : 1
                                    : 1
                                : 1
                            : 1
                        : 1
                    : 1
                : 1
            : 1
        : 1
    ) ? ((--vs)+4) ? print(ch) : (++vs) : {
        print(ch)
        vs = 0
    }
}

with the following variable assignments:
0   (unused)   (13 bytes)
1   cp         ( 4 bytes; occurs 20× in the code)
2   vs         ( 7 bytes; occurs  5× in the code)
3   ch         (10 bytes; occurs  3× in the code)

As you can see, I avoided variable slot 0 because 0 is such a long constant to write.
So we read each character and store the value in both cp and ch. We will modify cp but keep ch around so that we can print it if necessary. We successively subtract numbers 65, 4, 4, 6, etc. from cp to check if it’s each of the 10 possible vowel characters in ASCII (note the very last one doesn’t need to be an assignment).
vs always contains 3 less than the number of vowels still allowed to be printed. It starts out at 0, so 3 vowels can be printed. When it reaches -3, we stop printing vowels.
If we encounter a non-vowel (including the space), we execute print(ch) followed by vs = 0. As you have probably guessed, this resets the vowels counter.
If we encounter a vowel, we execute ((--vs)+4) ? print(ch) : (++vs). Let’s break this down:

decrement vs;
if the value is now -4, we’ve gone too far, so don’t print anything, but increment vs back to -3 so we will continue to refuse to print vowels;
otherwise, print the character.

This language is true to its name.

I always wonder in these languages... "Did they actually write this out by hand? If so, I pity them..." +1

Retina, 25 bytes
i`([aeiou]{3})[aeiou]+
$1

Try it online.
Fairly straightforward regex substitution. This also works for the same byte count:
Ri`(?<=[aeiou]{3})[aeiou]

Finally! An online interpreter! You should consider linking to it on your github page.

JavaScript (ES6), 42
As an anonymous function
s=>s.replace(/[aeiou]+/gi,v=>v.slice(0,3))

Perl, 27 characters
(26 characters code + 1 character command line option)
s/[aeiou]{3}\K[aeiou]+//gi

Not a big deal, just a rare occasion I remember \K exists.
Sample run:
bash-4.3$ perl -pe 's/[aeiou]{3}\K[aeiou]+//gi' <<< "
> Aeiou
> screeeen
> We're queueing up for the Hawaiian movie.
> Spaces break runs: aei iou."

Aei
screeen
We're queung up for the Hawaiin movie.
Spaces break runs: aei iou.

When I wrote up the Retina answer I thought "I wish .NET regex had \K". :)

Interesting, @MartinBüttner. I had the feeling those regular expressions were put on serious steroid diet. For curiosity, do they have recursive subpattern? Can help to spare one vowel enumeration, though the result is longer: s/([aeiou]{1,3})(?1)+/$1/gi.

Unfortunately, they don't have pattern reuse either. Those are the two things that occasionally make me switch to Perl or PCRE. When I get around to patching some simple things into Retina's regex flavour, I think I'll add those (not true recursion, but at least pattern reuse and finite recursion).

APL, 40 chars
{⍵/⍨1↓4≠⊃+/(1-⍳4)⌽¨⊂'aeiouAEIOU'∊⍨' ',⍵}

In English:

'aeiouAEIOU'∊⍨' ',⍵: find the vowels (and prepend a space to break on rotation);
(1-⍳4)⌽¨⊂: rotate 0, 1, 2, 3 times (with wrap-around) pushing to the right the boolean vector;
⊃+/ sum: the rotations and unbox
1↓4≠: find the different than 4, and remove the first one (to cather for the space we prepended)
⍵/⍨: in the argument, keep only the element where the sum was different than 4.

Mathematica, 68 bytes
a=Characters@"aeiouAEIOU";StringReplace[#,b:a~Repeated~{3}~~a..:>b]&

The regex answer would be the same length, but who uses regex?

C, 166 bytes
not the shortest answer by far but nicely golfed I think..
#define V v[1][i]!=
#define P printf("%c",v[1][i]),j
j;main(i,v)char**v;{for(i=0;V 0;i++)(V 97&V 'e'&V 'i'&V 'o'&V 'u'&V 65&V 69&V 73&V 79&V 85)?P=0:j>3?j++:P++;}

test case:
$ a.exe "We're queueing up for the Hawaiian movie."

We're queung up for the Hawaiin movie.

$ wc -c vowels.c

166 vowels.c

Javascript ES6, 43 chars
s=>s.replace(/([aeiou]{3})[aeiou]*/gi,"$1")

Test:
f=s=>s.replace(/([aeiou]{3})[aeiou]*/gi,"$1")
;`"Aeiou" => "Aei"
"screeeen" => "screeen"
"We're queueing up for the Hawaiian movie." => "We're queung up for the Hawaiin movie."
"Spaces break runs: aei iou." => "Spaces break runs: aei iou."`
.replace(/"/g,"").split("\n").every(s=>f((s=s.split(" => "))[0])==s[1])

Java, 115 bytes
class a{public static void main(String[] a){System.out.println(a[0].replaceAll("(?i)([aeiou]{3})[aeiou]*","$1"));}}

Expects input as program parameter.
Unit test output:
Aei
screeen
We're queung up for the Hawaiin movie.

Save one byte by removing the space between String[] and a. String[]a

Save 2 bytes by using print rather than println. I don't believe the spec requires a trailing newline.

Arturo, 44 bytes
$=>[replace&{/(?i)([aeiou]{3})[aeiou]+}"$1"]

Try it

Japt, 9 bytes
r"%v+"È¯3

Try it
r"%v+"È¯3     :Implicit input of string
r             :Replace
 "%v+"        :RegEx /[aeiou]+/gi
      È       :Pass each match through the following function
       ¯3     :  Slice to length 3

Vyxal s, 60 bits1, 7.5 bytes
⁽AḊƛAa[3Ẏ

Try it Online!

Tcl, 82 56 bytes
Saved 26 bytes thanks to the comment of @naffetS

Golfed version. Try it online!
proc f s {regsub -all {(?i)([aeiou]{3})[aeiou]*} $s \\1}

Ungolfed version. Try it online!
proc remove_vowels {s} {
    regsub -all -- {([aeiouAEIOU]{1,3})[aeiouAEIOU]*} $s {\1} result
    return $result
}

puts [remove_vowels "Aeiou"]
puts [remove_vowels "screeeen"]
puts [remove_vowels "We're queueing up for the Hawaiian movie."]
puts [remove_vowels "Spaces break runs: aei iou."]

# "Aeiou" => "Aei"
# "screeeen" => "screeen"
# "We're queueing up for the Hawaiian movie." => "We're queung up for the Hawaiin movie."
# "Spaces break runs: aei iou." => "Spaces break runs: aei iou."

Try it online! 56 bytes

Thunno 2 Ṡ, 24 bytes
Oı"([%]+)"kWDL+%ḄıDhṃ?3ɱ

Attempt This Online!
Feel like maybe I'm doing something wrong
Explained
Oı"([%]+)"kWDL+%ḄıDhṃ?3ɱ
O                        # Split the input on spaces
 ı                       # to each item in that:
  "([%]+)"kWDL+%Ḅ        #   split on the regex ([AEIOUaeiou]+) - the () makes it so that the group is retained
                 ı       #   to each group:
                  Dhṃ?   #     if it starts with a vowel:
                      3ɱ #       keep only the first three items
# Ṡ flag joins on spaces before printing

In 2.2.0, 11 bytes with J flag: µñṃıDJṃh?3ɱ

Perl 6,  36   35 bytes

{S:g:i/(<[aeiou]>**3)<[aeiou]>+/$0/} # 36 bytes

$ perl6 -pe 's:g:i/(<[aeiou]>**3)<[aeiou]>+/$0/' # 34 + 1 = 35 bytes

usage:
$ perl6 -pe 's:g:i/(<[aeiou]>**3)<[aeiou]>+/$0/' <<< "
> Aeiou
> screeeen
> We're queueing up for the Hawaiian movie.
> Spaces break runs: aei iou."

Aei
screeen
We're queung up for the Hawaiin movie.
Spaces break runs: aei iou.

C (205 bytes)
#include <stdio.h>
#define T(x)for(i=0;i<10;++i){if(v[i]==x){b=x;m=1;break;}}putchar(c);
main(b,c,i,m){char v[]="aeiouAEIOU";
while((c=getchar())!=EOF){if(!m){T(c);}else{if(b==c)continue;else{m=0;T(c);}}}}

(One line break added for clarity)

136 bytes

Matlab / Octave, 54 bytes
@(s)regexprep(s,'(?<=[aeiouAEIOU]{3})[aeiouAEIOU]','')

Example:
>> @(s)regexprep(s,'(?<=[aeiouAEIOU]{3})[aeiouAEIOU]','')
ans =
    @(s)regexprep(s,'(?<=[aeiouAEIOU]{3})[aeiouAEIOU]','')

>> ans('We''re queueing up for the Hawaiian movie.')
ans =
We're queung up for the Hawaiin movie.

Try it at ideone.

x86 MS-DOS .COM file, 44 bytes 36 bytes
.COM files are widely supported from MS-DOS 1 through the present---I'm running in dosemu, using 8086 commands only.
Reduced from 44 to 36 bytes by using REPNE SCASB to test for vowels instead of using a separate command to test each vowel.
Hex dump, reversible using `xxd -r -seek -256`:
0100: b3 03 43 b4 08 cd 21 88 c2 24 df b1 05 bf 1f 01   ..C...!..$......
0110: f2 ae 74 02 b3 05 4b 74 e9 b4 02 cd 21 eb e4 41   ..t...Kt....!..A
0120: 45 49 4f 55                                       EIOU

Unassembled using debug:
0100 B303    MOV BL,03     ; initialize counter to 3 (will increment by 1 to be 4)
0102 43      INC BX        ; increment counter--runs each time it hits 0 so it never goes <0
0103 B408    MOV AH,08     ;
0105 CD21    INT 21        ; with AH=8, read 1 char without echo
0107 88C2    MOV DL,AL     ; copy input for potential output
0109 24DF    AND AL,DF     ; make input uppercase for testing
010B B105    MOV CL,05     ; count of 5 vowels to test against
010D BF1F01  MOV DI,011F   ; location of first vowel to test against
0110 F2AE    REPNE SCASB   ; test input against each vowel
0112 7402    JZ 0116       ; if input was not a vowel:
0114 B305    MOV BL,05     ;    reset counter to 5 (will decrement by 1 to be 4)
0116 4B      DEC BX        ; decrement counter regardless
0117 74E9    JZ 0102       ; if hit 0 (fourth or later vowel): goto 102
0119 B402    MOV AH,02     ;
011B CD21    INT 21        ; with AH=2, print char
011D EBE4    JMP 0103      ; go to 103 for next character

bytes 011f-0123 contain the uppercase vowels AEIOU

Scala, 107 bytes
readLine.foldLeft("",0)((a,n)=>if(!"aeiou".contains(n|32))a._1+n->0 else if(a._2>2)a else(a._1+n,a._2+1))_1

Seriously, 34 bytes
,;ù0╗`Ok"aeiou"Okd-Y;╜+*;╗4>`M@░εj

Hex Dump:
2c3b9730bb604f6b226165696f75224f6b
642d593bbd2b2a3bbb343e604d40b0ee6a

Try it online
It uses the same algorithm as the Pyth answer, mapping over the string while keeping track of the length of the current run of vowels in a register, incrementing it whenever the current character is a vowel, and checking whether it has exceeded the allowed length, returning 0 if so, and then filtering the original string with this generated filter. It will be a lot shorter once we can use set subtraction on strings. (The Ok can be deleted and the Okd can be replaced with just @). I hear this feature is coming in the next update....

V, 21 bytes (noncompeting)
ñ[aeiou]ñÍãqû3}úsq*

Try it online!
Explanation:
ñ[aeiou]ñ                     "Assign the string `[aeiou]` to register 'q'
         Íã                   "Search and replace on multiple lines (case insensitive):
           <C-r>q             "Register 'q'
                 û3}          "Repeated 3 times
                    ús        "Mark the following to be removed:
                      <C-r>q* "Register 'q' repeated any number of times

This is just barely shorter than the more straightforward solution:
Íã[aeiou]û3}ús[aeiou]*

(22 bytes)

Jelly, 12 11 bytes
ḟ€ØckŻḣ€4FḊ

Try it online!
Copy-paste from my answer to a similar challenge, except somehow even worse. e€Øc‘×¥\<4x@ is nicer for one more byte (check edit history for an explanation).
 €             For each character of the input,
ḟ              filter out
  Øc           vowels.
     Ż         Prepend a 0 to the input, and
    k          partition that after truthy (nonempty) positions in the original.
      ḣ€4      Keep the first 4 elements of each slice,
         F     flatten the slices,
          Ḋ    and remove the 0.

Ruby, 44 bytes
><<$<.read.gsub(/([aeiou]{3})[aeiou]+/i,'\1')

Example:
% ruby -e "$><<$<.read.gsub(/([aeiou]{3})[aeiou]+/i,'\1')" <<< "
Aeiou
screeeen
We're queueing up for the Hawaiian movie.
Spaces break runs: aei iou."

Aei
screeen
We're queung up for the Hawaiin movie.
Spaces break runs: aei iou.

You wrote it: “Input is a string containing only printable ASCII character (0x20 to 0x7E, inclusive).” Then why spending extra characters with $<.read to make it handle multiline input (thus containing out of range character 0x0a) instead of gets?

@manatwork that's a really good point, thank you! Think it might save 2-3 bytes :)


text:
Product over exclusive and inclusive ranges
Inspired by this question by @CᴏɴᴏʀO'Bʀɪᴇɴ.
Taken from the question:

Your task is simple: given two integers a and b, output ∏[a,b]; that
  is, the product of the range between a and b. You may take a and b in
  any reasonable format, whether that be arguments to a function, a list
  input, STDIN, et cetera. You may output in any reasonable format, such
  as a return value (for functions) or STDOUT. a will always be less
  than b.
Note that the end may be exclusive or inclusive of b. I'm not picky.
  ^_^

The difference for this challenge is we are going to be picky about the range type. Input is a string of the form [a,b], (a,b], [a,b), or (a,b) where a [] is an inclusive boundary and () is an exclusive boundary. Given the explicit boundaries, provide the product of the range. Also the input range will always include at least 1 number, meaning ranges like (3,4) are invalid and need not be tested.
Test cases
[a,b) => result
[2,5) => 24
[5,10) => 15120
[-4,3) => 0
[0,3) => 0
[-4,0) => 24

[a,b] => result
[2,5] => 120
[5,10] => 151200
[-4,3] => 0
[0,3] => 0
[-4,-1] => 24

(a,b] => result
(2,5] => 60
(5,10] => 30240
(-4,3] => 0
(0,3] => 6
(-4,-1] => -6

(a,b) => result
(2,5) => 12
(5,10) => 3024
(-4,3) => 0
(0,3) => 2
(-4,0) => -6

This is a code-golf, so the shortest program in bytes wins.

Leaderboard
The Stack Snippet at the bottom of this post generates the catalog from the answers a) as a list of shortest solution per language and b) as an overall leaderboard.
To make sure that your answer shows up, please start your answer with a headline, using the following Markdown template:
## Language Name, N bytes

where N is the size of your submission. If you improve your score, you can keep old scores in the headline, by striking them through. For instance:
## Ruby, <s>104</s> <s>101</s> 96 bytes

If there you want to include multiple numbers in your header (e.g. because your score is the sum of two files or you want to list interpreter flag penalties separately), make sure that the actual score is the last number in the header:
## Perl, 43 + 2 (-p flag) = 45 bytes

You can also make the language name a link which will then show up in the snippet:
## [><>](http://esolangs.org/wiki/Fish), 121 bytes

var QUESTION_ID=66285,OVERRIDE_USER=44713;function answersUrl(e){return"https://api.stackexchange.com/2.2/questions/"+QUESTION_ID+"/answers?page="+e+"&pagesize=100&order=desc&sort=creation&site=codegolf&filter="+ANSWER_FILTER}function commentUrl(e,s){return"https://api.stackexchange.com/2.2/answers/"+s.join(";")+"/comments?page="+e+"&pagesize=100&order=desc&sort=creation&site=codegolf&filter="+COMMENT_FILTER}function getAnswers(){jQuery.ajax({url:answersUrl(answer_page++),method:"get",dataType:"jsonp",crossDomain:!0,success:function(e){answers.push.apply(answers,e.items),answers_hash=[],answer_ids=[],e.items.forEach(function(e){e.comments=[];var s=+e.share_link.match(/\d+/);answer_ids.push(s),answers_hash[s]=e}),e.has_more||(more_answers=!1),comment_page=1,getComments()}})}function getComments(){jQuery.ajax({url:commentUrl(comment_page++,answer_ids),method:"get",dataType:"jsonp",crossDomain:!0,success:function(e){e.items.forEach(function(e){e.owner.user_id===OVERRIDE_USER&&answers_hash[e.post_id].comments.push(e)}),e.has_more?getComments():more_answers?getAnswers():process()}})}function getAuthorName(e){return e.owner.display_name}function process(){var e=[];answers.forEach(function(s){var r=s.body;s.comments.forEach(function(e){OVERRIDE_REG.test(e.body)&&(r="
"+e.body.replace(OVERRIDE_REG,"")+"
")});var a=r.match(SCORE_REG);a&&e.push({user:getAuthorName(s),size:+a[2],language:a[1],link:s.share_link})}),e.sort(function(e,s){var r=e.size,a=s.size;return r-a});var s={},r=1,a=null,n=1;e.forEach(function(e){e.size!=a&&(n=r),a=e.size,++r;var t=jQuery("#answer-template").html();t=t.replace("{{PLACE}}",n+".").replace("{{NAME}}",e.user).replace("{{LANGUAGE}}",e.language).replace("{{SIZE}}",e.size).replace("{{LINK}}",e.link),t=jQuery(t),jQuery("#answers").append(t);var o=e.language;/s.lang?1:e.lang<s.lang?-1:0});for(var c=0;c<t.length;++c){var i=jQuery("#language-template").html(),o=t[c];i=i.replace("{{LANGUAGE}}",o.lang).replace("{{NAME}}",o.user).replace("{{SIZE}}",o.size).replace("{{LINK}}",o.link),i=jQuery(i),jQuery("#languages").append(i)}}var ANSWER_FILTER="!t)IWYnsLAZle2tQ3KqrVveCRJfxcRLe",COMMENT_FILTER="!)Q2B_A2kjfAiU78X(md6BoYk",answers=[],answers_hash,answer_ids,answer_page=1,more_answers=!0,comment_page;getAnswers();var SCORE_REG=/<h\d>\s*([^\n,]*[^\s,]),.*?(\d+)(?=[^\n\d<>]*(?:<(?:s>[^\n<>]*<\/s>|[^\n<>]+>)[^\n\d<>]*)*<\/h\d>)/,OVERRIDE_REG=/^Override\s*header:\s*/i;
body{text-align:left!important}#answer-list,#language-list{padding:10px;width:290px;float:left}table thead{font-weight:700}table td{padding:5px}
 <link rel="stylesheet" type="text/css" href="//cdn.sstatic.net/codegolf/all.css?v=83c949450c8b">

Leaderboard

	Author	Language	Size

Winners by Language

Language	User	Score

{{PLACE}}	{{NAME}}	{{LANGUAGE}}	{{SIZE}}	Link

{{LANGUAGE}}	{{NAME}}	{{SIZE}}	Link

LabVIEW, 38 LabVIEW Primitives
"slightly" modified, now sets the ranges by scanning for () and [] and adding the index to the numbers.

By having a language that requires a fancy gif, you have immediately gained ∞ rep. GG. +1

Vyxal, 24,  18 bytes
₌≬k+iE₍ht‛(]f=+÷rΠ

Try it Online!
I'm actually really proud of this answer, because it uses the new modifier parsing of 2.6.
With syntax highlighting:

Explained
₌≬k+iE₍ht‛(]f=+÷rΠ
₌                  # Apply the following to the same stack:
 ≬                 #   The next 3 elements as a single lambda, taking argument n:
  k+iE             #     eval(n[1:-1]) - the two numbers separated by a comma
₌                  #   and
      ₍ht          #   [n[0], n[-1]] - the range brackets
         ‛(]f=     # Does the first bracket equal "(" and does the last bracket equal "]" - this determines how much to offset the range; because the two numbers in the original input will be passed to Vyxal's range function which acts like python's range() function
              +    # add those offsets: an exclusive range start means the first number needs to be incremented because range() includes the first argument. an inclusive range start means the last number needs to be incremented because range() stops before the last argument.
               ÷r  # push the range between the two numbers + their offsets
                 Π # and take the product

Minecraft 15w35a+, program size 638 total (see below)
Same as my answer here, but modified. Since Minecraft has no string input, I took the liberty of keeping scoreboard input. If that is a problem, consider this answer non-competitive.

This calculates PI a,b with inclusive / exclusive specified by the two levers. Input is given by using these two commands: /scoreboard players set A A {num} and /scoreboard players set B A {num}. Remember to use /scoreboard objectives add A dummy before input.
Scored using: {program size} + ( 2 * {input command} ) + {scoreboard command} = 538 + ( 2 * 33 ) + 34 = 638.
This code corresponds to the following psuedocode:
R = 1
T = A
loop:
  R *= A
  A += 1
  if A == B:
    if A.exclusive:
      R /= T
    if B.exclusive:
      R /= B
    print R
    end program

Download the world here.

Python 2, 72 bytes
lambda s:reduce(int.__mul__,range(*eval(s[1:-1]+'+'+`']'in s`))[s<'[':])

To extract the numbers we evaluate s[1:-1], the input string with ends  removed, which gives a tuple. The idea is to get the range of this tuple and take the product.
lambda s:reduce(int.__mul__,range(*eval(s[1:-1]))

The fudging happens to adjust the endpoints. The upper endpoint is easy, just cut off the first element if the input starts with (, done as [s<'[':].
The other endpoint is trickier. Python doesn't have a clean way to conditionally remove the last element of a list because l[:0] removes the whole thing. So, we do something weird. We modify the tuple string before it is evaluated to tack on the string "+True" or "+False" depending on whether s ends in ] or ). The result is that something like 3,7 becomes either 3,7+False which is 3,7, or 3,7+True which is 3,8.
Alternate, prettier 72:
lambda s:eval("reduce(int.__mul__,range((s<'[')+%s+(']'in s)))"%s[1:-1])

Julia 0.4, 75 bytes
s->prod((x=map(parse,split(s[2:end-1],",")))[1]+(s[1]<41):x[2]-(s[end]<42))

Try it online!
This is an anonymous function that accepts a string and returns an integer. To call it, give it a name, e.g. f=s->....
Ungolfed:
function f(s::AbstractString)
    # Extract the numbers in the input
    x = map(parse, split(s[2:end-1], ","))

    # Construct a range, incrementing or decrementing the endpoints
    # based on the ASCII value of the surrounding bracket
    r = x[1]+(s[1] == 40):x[2]-(s[end] == 41)

    # Return the product over the range
    return prod(r)
end

Pyth, 20 bytes
*FPW}\)ztW}\(z}FvtPz

Try it online: Demonstration or Test Suite
Explanation:
*FPW}\)ztW}\(z}FvtPz   implicit: z = input string
                 tPz   remove the first and last character of z
                v      evaluate, returns a tuple of numbers
              }F       inclusive range
        tW             remove the first number, if
          }\(z            "(" in z
  PW                   remove the last number, if
    }\)z                  ")" in z
*F                     compute the product of the remaining numbers

Ruby, 79 77 bytes
->s{a,b=s.scan /\-?\d+/;(a.to_i+(s[?[]?0:1)..b.to_i-(s[?]]?0:1)).reduce 1,:*}

79 bytes
->s{a,b=s.scan(/\-?\d+/).map &:to_i;((s[?[]?a:a+1)..(s[?]]?b:b-1)).reduce 1,:*}

Ungolfed:
-> s {
  a,b=s.scan /\-?\d+/    # Extracts integers from the input string, s
  (
    a.to_i+(s[?[]?0:1).. # Increase start of the range by 1 if s contains `(`
    b.to_i-(s[?]]?0:1)   # Decrease end of the range by 1 if s contains `)`
  ).reduce 1,:*
}

Usage:
->s{a,b=s.scan /\-?\d+/;(a.to_i+(s[?[]?0:1)..b.to_i-(s[?]]?0:1)).reduce 1,:*}["(2,5]"]
=> 60

Seriously, 31 bytes
,#d@p@',@s`εj≈`Mi(@)']=+)'(=+xπ

Takes input as a string (wrapped in double quotes)
Try it online (input must be manually entered)
Explanation:
,#d@p@                             get input, take first and last character off and push them individually
      ',@s`εj≈`Mi                  split on commas, map: join on empty, cast to int; explode list
                 (@)']=+)'(=+      increment start and end if braces are ( and ] respectively (since range does [a,b))
                             xπ    make range, push product

JavaScript (ES6), 68 bytes
s=>(f=_=>a>b||a++*f())([a,b,c]=s.match(/-?\d+|]/g),a-=-(s<'['),b-=!c)

Welcome to Code Golf, and nice answer!

Python 3, 104
y,r=input().split(',')
t=int(y[1:])+(y[0]<')')
for x in range(t+1,int(r[:-1])+(r[-1]>'[')):t*=x
print(t)

Takes input from stdin.

we actually posted our answers in the same second O.o

@Eumel That should be a badge.

ill actually post that on Meta right now^^

@Eumel: Actually you posted yours 1 second before Morgan Thrapp's

oh really? it showed answered n seconds ago on both answers

MATLAB, 86 70 bytes
s=sscanf(input(''),'%c%d,%d%c');a=s<42;disp(prod(a(1)+s(2):s(3)-a(4)))

This also works with Octave. You can try online here. I have added the code as a script to that workspace, so you can just enter productRange at the prompt, then enter your input, e.g. '(2,5]'.

So the code first scans the input to extract the brackets and the numbers together:
s=sscanf(input(''),'%c%d,%d%c');

This returns an array which is made of [bracket, number, number, bracket].
The array is compared with 42, actually any number between 42 and 90 inclusive will do. This determines which sort of bracket it was, giving a 1 if it is an exclusive bracket, and a 0 if an inclusive bracket.
a=s<42;

Finally, we display the product of the required range:
disp(prod(a(1)+s(2):s(3)-a(4)))

The product is of numbers staring with the first number s(2) plus the first bracket type a(1) (which is a 1 if an exclusive bracket), ranging up to and including the second number s(3) minus the second bracket type a(4). This gives the correct inclusive/exclusive range.

PowerShell, 146 104 Bytes
param($i)$a,$b=$i.trim("[]()")-split',';($a,(1+$a))[$i[0]-eq'(']..($b,(+$b-1))[$i[-1]-eq')']-join'*'|iex

Golfed off 42 bytes by changing how the numbers are extracted from the input. Woo!
param($i)                          # Takes input string as $i
$a,$b=$i.trim("[]()")-split','     # Trims the []() off $i, splits on comma,
                                   # stores the left in $a and the right in $b

($a,(1+$a))[$i[0]-eq'(']..($b,(+$b-1))[$i[-1]-eq')']-join'*'|iex
# Index into a dynamic array of either $a or $a+1 depending upon if the first
# character of our input string is a ( or not
# .. ranges that together with
# The same thing applied to $b, depending if the last character is ) or not
# Then that's joined with asterisks before
# Being executed (i.e., eval'd)

Perl 6, 60 bytes
{s/\((\-?\d+)/[$0^/;s/(\-?\d+)\)/^$0]/;s/\,/../;[*] EVAL $_}

There is a bit of mis-match because the way you would write the (2,5] example in Perl 6 would be 2^..5 ([2^..5] also works).
So I have to swap (2 with [2^, and , with .., then I have to EVAL it into a Range.

usage:
# give it a name
my &code = {...}

# the `$ =` is so that it gets a scalar instead of a constant

say code $ = '(2,5)'; # 12
say code $ = '[2,5)'; # 24
say code $ = '(2,5]'; # 60
say code $ = '[2,5]'; # 120

say code $ = '(-4,0)' # -6
say code $ = '[-4,0)' # 24
say code $ = '(-4,0]' # 0
say code $ = '[-4,0]' # 0

say code $ = '(-4,-1)' # 6
say code $ = '[-4,-1)' # -24
say code $ = '(-4,-1]' # -6
say code $ = '[-4,-1]' # 24

# this is perfectly cromulent,
# as it returns the identity of `*`
say code $ = '(3,4)'; # 1

Mathematica, 128 bytes
1##&@@Range[(t=ToExpression)[""<>Rest@#]+Boole[#[[1]]=="("],t[""<>Most@#2]-Boole[Last@#2==")"]]&@@Characters/@#~StringSplit~","&

This is too long... Currently thinking about a StringReplace + RegularExpression solution.

JavaScript (ES6), 90 bytes
s=>eval(`for(n=s.match(/-*\\d+/g),i=n[0],c=s[0]<"["||i;++i<+n[1]+(s.slice(-1)>")");)c*=i`)

Explanation
s=>
  eval(`                    // use eval to allow for loop without return or {}
    for(
      n=s.match(/-*\\d+/g), // n = array of input numbers [ a, b ]
      i=n[0],               // i = current number to multiply the result by
      c=s[0]<"["||i;        // c = result, initialise to a if inclusive else 1
      ++i<+n[1]             // iterate from a to b
        +(s.slice(-1)>")"); // if the end is inclusive, increment 1 more time
    )
      c*=i                  // multiply result
  `)                        // implicit: return c

Test

var solution = s=>eval(`for(n=s.match(/-*\\d+/g),i=n[0],c=s[0]<"["||i;++i<+n[1]+(s.slice(-1)>")");)c*=i`)
<input type="text" id="input" value="(5,10]" />
<button onclick="result.textContent=solution(input.value)">Go</button>

TI-Basic, 69 bytes
Input Str1
expr("{"+sub(Str1,2,length(Str1)-2
prod(randIntNoRep(min(Ans)+(sub(Str1,1,1)="("),max(Ans)-(sub(Str1,length(Str1),1)=")

Output is stored in Ans and displayed. Replace randIntNoRep( with seq(I,I, (+3 bytes) if the calculator does not support it.

Wolfram Language (Mathematica), 57 bytes
1##&@@ToExpression["Range"<>(#/."("->"[1+"/.")"->"-1]")]&

Try it online!
Input a list of characters.
Fairly straightforward. Mathematica uses [ ] to introduce arguments, and Range is inclusive on both ends.

Wolfram Language (Mathematica), 69 bytes
1##&@@ToExpression["Range"<>#~StringReplace~{"("->"[1+",")"->"-1]"}]&

Try it online!
Input a string.

Japt, 43 41 bytes
[VW]=Uf"\\d+";ÂV+Â('A>Ug¹oÂW+Â('A<UtJ¹r*1

Try it online!

R, 102 104 bytes
f=function(s){x=scan(t=gsub('\\(|\\[|,|\\)|\\]',' ',s))+c(grepl('^\\(',s),-(grepl('\\)$',s)));prod(x[1]:x[2])}

Ungolfed
f=function(s){
    # remove delimiting punctuation from input string, parse and return an atomic vector
    x=scan(t=gsub('\\(|\\[|,|\\)|\\]',' ',s)) +
    # add /subtract from the range dependent on the `[)` pre/suf-fixes
    c(grepl('^\\(',s),-(grepl('\\)$',s)))
    # get the product of the appropriate range of numbers
    prod(x[1]:x[2])
}

edit to allow negative numbers [at the expense of 2 more characters

Language?

@ThisSuitIsBlackNot --R (and fixed in answer)

CJam, 34 bytes
r)\(@+"[()]"2/\.#\',/:i.+~1$-,f+:*

Try it online
Explanation:
r       Read input.
)       Split off last character.
\       Swap rest of input to top.
(       Split off first character.
@       Rotate last character to top.
+       Concatenate first and last character, which are the two braces.
"[()]"  Push string with all possible braces.
2/      Split it into start and end braces.
\       Swap braces from input to top.
.#      Apply find operator to vector elements, getting the position of each brace
        from input in corresponding list of possible braces. The lists of braces
        are ordered so that the position of each can be used as an offset for the
        start/end value of the interval.
\       Swap remaining input, which is a string with two numbers separated by
        a comma, to top.
',/     Split it at comma.
:i      Convert the two values from string to integer.
.+      Element-wise addition to add the offsets based on the brace types.
~       Unwrap the final start/end values for the interval.
1$      Copy start value to top.
-       Subtract it from end value.
,       Build 0-based list of values with correct length.
f+      Add the start value to all values.
:*      Reduce with multiplication.

JavaScript (ES6), 79
As an anonymous method
r=>eval("[a,b,e]=r.match(/-?\\d+|.$/g);c=a-=-(r<'@');for(b-=e<'@';a++eval("[a,b,e]=r.match(/-?\\d+|.$/g);c=a-=-(r<'@');for(b-=e<'@';a++O.innerHTML+=x+'\n'

;[
 ['[2,5)',24],['[5,10)',15120],['[-4,3)',0],['[0,3)',0],['[-4,0)',24],
 ['[2,5]',120],['[5,10]',151200],['[-4,3]',0],['[0,3]',0],['[-4,-1]',24],
 ['(2,5]',60],['(5,10]',30240],['(-4,3]',0],['(0,3]',6],['(-4,-1]',-6],
 ['(2,5)',12],['(5,10)',3024],['(-4,3)',0],['(0,3)',2],['(-4,0)',-6]
].forEach(t=>{
  r=F(t[0]),k=t[1],console.log(t[0]+' -> '+r+' (check '+k+ (k==r?' ok)':' fail)'))
})

Japt, 25 bytes
Ugly as hell!
Å¯J q, rÈ°+Uø'()õY-Uø')Ã×

Try it (includes all test cases)

Haskell, 94 bytes
f=(\(h:a,_:b)->product[a!h..init b!last b]).span(','/=);c!')'=c!' '-1;c!'('=c!' '+1;c!_=read c

Try it online!

Pip, 20 bytes
$*VaR^"(][)"['U"+1"]

Replit! Or, verify all test cases with this 22-byte version in Pip Classic: Try it online!
Explanation
Conveniently, , is Pip's range operator; it is inclusive on the lower end and exclusive on the upper end. So if we turn the brackets into appropriate tweaks to the numbers, we can use eval to generate the range.
$*VaR^"(][)"['U"+1"]
   aR                 In the input, replace each string in
      "(][)"           This string
     ^                 Split into a list of characters
                      with the corresponding string in
            [      ]  this list:
             'U        ( becomes U (increment the following number)
               "+1"    ] becomes +1 (add 1 to the preceding number)
                       [ and ) don't have a corresponding replacement, so they are
                       simply deleted (leaving the numbers unchanged)
  V                   Evaluate the result as a Pip expression
$*                    Fold on multiplication

Julia 0.4, 64 bytes
s->prod((x=parse(s[2:end-1]).args)[]+(s[1]<41):x[2]-(s[end]<42))

Try it online!
Based on Alex A's answer
Julia 1.0, 69 bytes
s->prod((:)(Meta.parse(s[2:end-1]).args+[s[1]<'*',-(s[end]<'*')]...))

Try it online!


text:
Priming a Pristine World
Heavily inspired by Programming a Pristine World. Also closely related to this challenge.

Let's define a pristine prime as a number which is itself prime, but will no longer be prime if you remove any contiguous substring of N base 10 digits, where 0 < N < digits in number.
For example, 409 is a pristine prime because 409 itself is prime, but all numbers resulting from removing a substring of 1 digit are not prime:
40
49
09 = 9

and all numbers resulting from removing substrings of length 2 are not prime:
4
9

On the other hand, the prime number 439 is not pristine. Removing the different substrings results in:
43
49
39
4
9

While 49, 39, 4, and 9 are all non-prime, 43 is prime; thus, 439 is not pristine.
2, 3, 5, and 7 are trivially pristine, since they cannot have any substrings removed.
Challenge
Your challenge is to create a program or function that takes in a positive integer N and outputs the Nth pristine prime. The code should finish in under 1 minute on any modern PC for any input up to 50.
The shortest code in bytes wins.
As a reference, here are the first 20 pristine primes:
N    Pristine prime
1    2
2    3
3    5
4    7
5    11
6    19
7    41
8    61
9    89
10   409
11   449
12   499
13   821
14   881
15   991
16   6299
17   6469
18   6869
19   6899
20   6949

Here is a full list of pristine primes up to 1e7, or N = 376.
Finally, here are two related OEIS entries:

A033274: very similar, but generated by keeping substrings instead of removing them.
A071062: oddly similar, but generated in a much different manner.

Very closely related

Pyth, 29 bytes
e.f>}ZPZsmq1lPs.D`Z}Fd.CU`Z2Q

Golfing, explanation, etc. to follow.

…and it never followed :(

CJam, 51 bytes
1ri{{)_mp1$s_,)2m*{:>},\f{\~2$<@@>+0e|imp}1b!&!}g}*

Just a first pass, this can probably be improved a lot.
Test it here.

Japt, 61 bytes
$while(V<U)T$°,W=Ts ,Tj «Wl o d@1o1-X+Wl)dZ{WjYZ n j} } ©V°;T

Try it online!
It's a shame I haven't implemented loops in Japt yet, otherwise this would be a good bit shorter. Still golfing...